v1.2.2
Co-Authored-By: Quentin Torroba <quentin.torroba@mistral.ai> Co-Authored-By: Michel Thomazo <michel.thomazo@mistral.ai>
This commit is contained in:
Mathias Gesbert
Mathias Gesbert
parent
402e898f39
commit
2e1e15120d
32 changed files with 391 additions and 549 deletions
|
|
@ -159,9 +159,7 @@ class TestAgentStatsHelpers:
|
|||
class TestReloadPreservesStats:
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_session_tokens(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="First response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="First response"))
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
|
|
@ -185,13 +183,14 @@ class TestReloadPreservesStats:
|
|||
tool_calls=[
|
||||
ToolCall(
|
||||
id="tc1",
|
||||
index=0,
|
||||
function=FunctionCall(
|
||||
name="todo", arguments='{"action": "read"}'
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
mock_llm_chunk(content="Done", finish_reason="stop"),
|
||||
mock_llm_chunk(content="Done"),
|
||||
])
|
||||
config = make_config(enabled_tools=["todo"])
|
||||
agent = Agent(config, mode=AgentMode.AUTO_APPROVE, backend=backend)
|
||||
|
|
@ -210,8 +209,8 @@ class TestReloadPreservesStats:
|
|||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_steps(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="R1", finish_reason="stop"),
|
||||
mock_llm_chunk(content="R2", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="R1")],
|
||||
[mock_llm_chunk(content="R2")],
|
||||
])
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
|
|
@ -231,9 +230,7 @@ class TestReloadPreservesStats:
|
|||
async def test_reload_preserves_context_tokens_when_messages_preserved(
|
||||
self,
|
||||
) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
assert agent.stats.context_tokens > 0
|
||||
|
|
@ -258,10 +255,10 @@ class TestReloadPreservesStats:
|
|||
assert agent.stats.context_tokens == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_context_tokens_when_messages_exist(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
async def test_reload_resets_context_tokens_when_system_prompt_changes(
|
||||
self,
|
||||
) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config1 = make_config(system_prompt_id="tests")
|
||||
config2 = make_config(system_prompt_id="cli")
|
||||
agent = Agent(config1, backend=backend)
|
||||
|
|
@ -279,9 +276,7 @@ class TestReloadPreservesStats:
|
|||
async def test_reload_updates_pricing_from_new_model(self, monkeypatch) -> None:
|
||||
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
|
||||
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config_mistral = make_config(active_model="devstral-latest")
|
||||
agent = Agent(config_mistral, backend=backend)
|
||||
|
||||
|
|
@ -302,8 +297,8 @@ class TestReloadPreservesStats:
|
|||
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
|
||||
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="First", finish_reason="stop"),
|
||||
mock_llm_chunk(content="After reload", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="First")],
|
||||
[mock_llm_chunk(content="After reload")],
|
||||
])
|
||||
config1 = make_config(active_model="devstral-latest")
|
||||
agent = Agent(config1, backend=backend)
|
||||
|
|
@ -330,9 +325,7 @@ class TestReloadPreservesStats:
|
|||
class TestReloadPreservesMessages:
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_conversation_messages(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
|
|
@ -353,9 +346,7 @@ class TestReloadPreservesMessages:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_updates_system_prompt_preserves_rest(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config1 = make_config(system_prompt_id="tests")
|
||||
agent = Agent(config1, backend=backend)
|
||||
|
||||
|
|
@ -388,9 +379,7 @@ class TestReloadPreservesMessages:
|
|||
self, observer_capture
|
||||
) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = Agent(make_config(), message_observer=observer, backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
|
|
@ -410,8 +399,8 @@ class TestCompactStatsHandling:
|
|||
@pytest.mark.asyncio
|
||||
async def test_compact_preserves_cumulative_stats(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="First response", finish_reason="stop"),
|
||||
mock_llm_chunk(content="<summary>", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="First response")],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
|
|
@ -432,8 +421,8 @@ class TestCompactStatsHandling:
|
|||
@pytest.mark.asyncio
|
||||
async def test_compact_updates_context_tokens(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Long response " * 100, finish_reason="stop"),
|
||||
mock_llm_chunk(content="<summary>", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="Long response " * 100)],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
|
|
@ -449,19 +438,22 @@ class TestCompactStatsHandling:
|
|||
@pytest.mark.asyncio
|
||||
async def test_compact_preserves_tool_call_stats(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(
|
||||
content="Using tool",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="tc1",
|
||||
function=FunctionCall(
|
||||
name="todo", arguments='{"action": "read"}'
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
mock_llm_chunk(content="Done", finish_reason="stop"),
|
||||
mock_llm_chunk(content="<summary>", finish_reason="stop"),
|
||||
[
|
||||
mock_llm_chunk(
|
||||
content="Using tool",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="tc1",
|
||||
index=0,
|
||||
function=FunctionCall(
|
||||
name="todo", arguments='{"action": "read"}'
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
mock_llm_chunk(content=" todo"),
|
||||
],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
config = make_config(enabled_tools=["todo"])
|
||||
agent = Agent(config, mode=AgentMode.AUTO_APPROVE, backend=backend)
|
||||
|
|
@ -478,8 +470,8 @@ class TestCompactStatsHandling:
|
|||
@pytest.mark.asyncio
|
||||
async def test_compact_resets_session_id(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Long response " * 100, finish_reason="stop"),
|
||||
mock_llm_chunk(content="<summary>", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="Long response " * 100)],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
agent = Agent(make_config(disable_logging=False), backend=backend)
|
||||
|
||||
|
|
@ -506,8 +498,8 @@ class TestAutoCompactIntegration:
|
|||
observed.append((msg.role, msg.content))
|
||||
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="<summary>", finish_reason="stop"),
|
||||
mock_llm_chunk(content="<final>", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = VibeConfig(
|
||||
session_logging=SessionLoggingConfig(enabled=False),
|
||||
|
|
@ -544,9 +536,7 @@ class TestAutoCompactIntegration:
|
|||
class TestClearHistoryFullReset:
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_fully_resets_stats(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
|
|
@ -563,9 +553,7 @@ class TestClearHistoryFullReset:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_preserves_pricing(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config = make_config(input_price=0.4, output_price=2.0)
|
||||
agent = Agent(config, backend=backend)
|
||||
|
||||
|
|
@ -579,9 +567,7 @@ class TestClearHistoryFullReset:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_removes_messages(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
|
|
@ -596,9 +582,7 @@ class TestClearHistoryFullReset:
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_resets_session_id(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = Agent(make_config(disable_logging=False), backend=backend)
|
||||
|
||||
original_session_id = agent.session_id
|
||||
|
|
@ -622,9 +606,7 @@ class TestStatsEdgeCases:
|
|||
) -> None:
|
||||
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
|
||||
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Response", finish_reason="stop")
|
||||
])
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config1 = make_config(active_model="devstral-latest")
|
||||
agent = Agent(config1, backend=backend)
|
||||
|
||||
|
|
@ -643,9 +625,9 @@ class TestStatsEdgeCases:
|
|||
@pytest.mark.asyncio
|
||||
async def test_multiple_reloads_accumulate_correctly(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="R1", finish_reason="stop"),
|
||||
mock_llm_chunk(content="R2", finish_reason="stop"),
|
||||
mock_llm_chunk(content="R3", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="R1")],
|
||||
[mock_llm_chunk(content="R2")],
|
||||
[mock_llm_chunk(content="R3")],
|
||||
])
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
|
|
@ -668,9 +650,9 @@ class TestStatsEdgeCases:
|
|||
@pytest.mark.asyncio
|
||||
async def test_compact_then_reload_preserves_both(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Initial response", finish_reason="stop"),
|
||||
mock_llm_chunk(content="<summary>", finish_reason="stop"),
|
||||
mock_llm_chunk(content="After reload", finish_reason="stop"),
|
||||
[mock_llm_chunk(content="Initial response")],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="After reload")],
|
||||
])
|
||||
agent = Agent(make_config(), backend=backend)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue