v2.17.0 (#822)
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai> Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com> Co-authored-by: Hdandria <henri.dandria@mistral.ai> Co-authored-by: Ivana Dunisijevic <ivana.dunisijevic@mistral.ai> Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com> Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai> Co-authored-by: Mert Unsal <mert.unsal@mistral.ai> Co-authored-by: Michel Thomazo <51709227+michelTho@users.noreply.github.com> Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com> Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai> Co-authored-by: Val <102326092+vdeva@users.noreply.github.com> Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com> Co-authored-by: renovate-mistral[bot] <253709520+renovate-mistral[bot]@users.noreply.github.com> Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
parent
564a14365e
commit
6bedf271ce
223 changed files with 10533 additions and 6947 deletions
0
tests/agent_loop/e2e/__init__.py
Normal file
0
tests/agent_loop/e2e/__init__.py
Normal file
181
tests/agent_loop/e2e/conftest.py
Normal file
181
tests/agent_loop/e2e/conftest.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.constants import (
|
||||
ANTHROPIC_BASE_URL,
|
||||
ANTHROPIC_MESSAGES_PATH,
|
||||
CHAT_COMPLETIONS_PATH,
|
||||
CONNECTORS_BOOTSTRAP_PATH,
|
||||
MISTRAL_BASE_URL,
|
||||
OPENAI_BASE_URL,
|
||||
OPENAI_RESPONSES_PATH,
|
||||
)
|
||||
from vibe.core.agent_loop import AgentLoop
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
|
||||
from vibe.core.llm.backend.factory import BACKEND_FACTORY
|
||||
from vibe.core.types import AssistantEvent, Backend, BaseEvent
|
||||
|
||||
|
||||
def assistant_text(events: list[BaseEvent]) -> str:
|
||||
return "".join(
|
||||
e.content for e in events if isinstance(e, AssistantEvent) and e.content
|
||||
).strip()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _git_offline(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Restrict git to the local file protocol so no test can fetch/push over the
|
||||
# network, even if a remote is misconfigured to a real URL.
|
||||
monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file")
|
||||
|
||||
|
||||
def e2e_config(**overrides: Any) -> VibeConfig:
|
||||
# Defaults give the mistral provider/model (api.mistral.ai, the respx-mocked
|
||||
# base). The default model reasons, but that only adds reasoning_effort to the
|
||||
# request, which the mocked responses ignore.
|
||||
# The e2e harness mocks the connector bootstrap endpoint via respx, so
|
||||
# connector discovery is enabled (and fast) here unlike the unit-test default.
|
||||
return build_test_vibe_config(
|
||||
enabled_tools=overrides.pop("enabled_tools", []),
|
||||
enable_connectors=overrides.pop("enable_connectors", True),
|
||||
system_prompt_id="tests",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
def _generic_e2e_config(
|
||||
*, name: str, api_base: str, api_style: str, model: str, **overrides: Any
|
||||
) -> VibeConfig:
|
||||
provider = ProviderConfig(
|
||||
name=name,
|
||||
api_base=api_base,
|
||||
api_key_env_var=f"{name.upper()}_API_KEY",
|
||||
api_style=api_style,
|
||||
backend=Backend.GENERIC,
|
||||
)
|
||||
models = [ModelConfig(name=model, provider=name, alias=name)]
|
||||
return e2e_config(
|
||||
active_model=name, models=models, providers=[provider], **overrides
|
||||
)
|
||||
|
||||
|
||||
def anthropic_e2e_config(**overrides: Any) -> VibeConfig:
|
||||
return _generic_e2e_config(
|
||||
name="anthropic",
|
||||
api_base=ANTHROPIC_BASE_URL,
|
||||
api_style="anthropic",
|
||||
model="claude-test",
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
def openai_responses_e2e_config(**overrides: Any) -> VibeConfig:
|
||||
return _generic_e2e_config(
|
||||
name="openai",
|
||||
api_base=f"{OPENAI_BASE_URL}/v1",
|
||||
api_style="openai-responses",
|
||||
model="gpt-test",
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mistral() -> Iterator[respx.MockRouter]:
|
||||
with respx.mock(base_url=MISTRAL_BASE_URL, assert_all_called=False) as router:
|
||||
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
|
||||
return_value=httpx.Response(200, json={"connectors": []})
|
||||
)
|
||||
yield router
|
||||
|
||||
|
||||
def build_e2e_agent_loop(
|
||||
*, config: VibeConfig | None = None, enable_streaming: bool = False, **kwargs: Any
|
||||
) -> AgentLoop:
|
||||
resolved_config = config or e2e_config()
|
||||
provider = resolved_config.providers[0]
|
||||
# todo now we use build_test_agent_loop for the fake mcp registry.
|
||||
# Maybe mock http tool calls and instantiate a real registry later for more coverage
|
||||
return build_test_agent_loop(
|
||||
config=resolved_config,
|
||||
agent_name=kwargs.pop("agent_name", BuiltinAgentName.AUTO_APPROVE),
|
||||
backend=BACKEND_FACTORY[provider.backend](provider=provider),
|
||||
enable_streaming=enable_streaming,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class ProviderAPI:
|
||||
"""Stubs a provider's completion wire; the AgentLoop stays the subject."""
|
||||
|
||||
def __init__(self, router: respx.MockRouter, path: str) -> None:
|
||||
self.route = router.post(path)
|
||||
|
||||
def reply(self, *completions: dict[str, Any]) -> None:
|
||||
responses = [httpx.Response(200, json=c) for c in completions]
|
||||
if len(responses) == 1:
|
||||
self.route.mock(return_value=responses[0])
|
||||
else:
|
||||
self.route.mock(side_effect=responses)
|
||||
|
||||
def reply_stream(self, chunks: list[bytes]) -> None:
|
||||
self.route.mock(return_value=self._stream_response(chunks))
|
||||
|
||||
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
|
||||
self.route.mock(
|
||||
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stream_response(chunks: list[bytes]) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
)
|
||||
|
||||
@property
|
||||
def request_json(self) -> dict[str, Any]:
|
||||
return json.loads(self.route.calls.last.request.content)
|
||||
|
||||
|
||||
class MistralAPI(ProviderAPI):
|
||||
def __init__(self, router: respx.MockRouter) -> None:
|
||||
super().__init__(router, CHAT_COMPLETIONS_PATH)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mistral_api(mock_mistral: respx.MockRouter) -> MistralAPI:
|
||||
return MistralAPI(mock_mistral)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_anthropic() -> Iterator[respx.MockRouter]:
|
||||
with respx.mock(base_url=ANTHROPIC_BASE_URL, assert_all_called=False) as router:
|
||||
yield router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anthropic_api(mock_anthropic: respx.MockRouter) -> ProviderAPI:
|
||||
return ProviderAPI(mock_anthropic, ANTHROPIC_MESSAGES_PATH)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_openai() -> Iterator[respx.MockRouter]:
|
||||
with respx.mock(base_url=OPENAI_BASE_URL, assert_all_called=False) as router:
|
||||
yield router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_responses_api(mock_openai: respx.MockRouter) -> ProviderAPI:
|
||||
return ProviderAPI(mock_openai, OPENAI_RESPONSES_PATH)
|
||||
111
tests/agent_loop/e2e/test_e2e_agent_loop.py
Normal file
111
tests/agent_loop/e2e/test_e2e_agent_loop.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import (
|
||||
MistralAPI,
|
||||
assistant_text,
|
||||
build_e2e_agent_loop,
|
||||
e2e_config,
|
||||
)
|
||||
from tests.backend.data.mistral import (
|
||||
STREAMED_SIMPLE_CONVERSATION_PARAMS,
|
||||
mistral_completion,
|
||||
)
|
||||
from vibe.core.types import (
|
||||
AssistantEvent,
|
||||
ToolCallEvent,
|
||||
ToolResultEvent,
|
||||
UserMessageEvent,
|
||||
)
|
||||
|
||||
TODO_TOOL_CALL = [
|
||||
{
|
||||
"id": "call_todo_1",
|
||||
"function": {"name": "todo", "arguments": '{"action": "read"}'},
|
||||
"index": 0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _chunked_completion(reasoning: str, answer: str) -> dict[str, Any]:
|
||||
completion = mistral_completion(answer)
|
||||
completion["choices"][0]["message"]["content"] = [
|
||||
{"type": "thinking", "thinking": [{"type": "text", "text": reasoning}]},
|
||||
{"type": "text", "text": answer},
|
||||
]
|
||||
return completion
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_completes_against_real_backend(mistral_api: MistralAPI) -> None:
|
||||
# A plain prompt yields a user event then the backend's assistant reply, and
|
||||
# the loop accumulates prompt+completion tokens from the usage block.
|
||||
mistral_api.reply(
|
||||
mistral_completion("Hi there!", prompt_tokens=120, completion_tokens=30)
|
||||
)
|
||||
agent = build_e2e_agent_loop()
|
||||
|
||||
events = [event async for event in agent.act("Hello")]
|
||||
|
||||
assert [type(e) for e in events] == [UserMessageEvent, AssistantEvent]
|
||||
assert isinstance(events[1], AssistantEvent)
|
||||
assert events[1].content == "Hi there!"
|
||||
assert agent.stats.context_tokens == 150
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_streaming(mistral_api: MistralAPI) -> None:
|
||||
# Streamed SSE chunks are reassembled into the final assistant content.
|
||||
_, chunks, _ = STREAMED_SIMPLE_CONVERSATION_PARAMS[0]
|
||||
mistral_api.reply_stream(chunks)
|
||||
agent = build_e2e_agent_loop(enable_streaming=True)
|
||||
|
||||
events = [event async for event in agent.act("Hi")]
|
||||
|
||||
assert assistant_text(events).endswith("Some content")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_tool_call_round_trip(mistral_api: MistralAPI) -> None:
|
||||
# A tool call is parsed, executed, and its result fed back for a final reply.
|
||||
mistral_api.reply(
|
||||
mistral_completion("", tool_calls=TODO_TOOL_CALL),
|
||||
mistral_completion("All done"),
|
||||
)
|
||||
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
|
||||
|
||||
events = [event async for event in agent.act("Show my todos")]
|
||||
|
||||
assert any(isinstance(e, ToolCallEvent) for e in events)
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
final = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assert final and final[-1].content == "All done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_serializes_tools_in_request_payload(mistral_api: MistralAPI) -> None:
|
||||
# Enabled tools are serialized into the outgoing chat-completions request.
|
||||
mistral_api.reply(mistral_completion("ok"))
|
||||
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
|
||||
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
names = {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
|
||||
assert "todo" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_extracts_text_from_chunked_content_array(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# A thinking+text content array has its text part extracted as the reply.
|
||||
mistral_api.reply(_chunked_completion("thinking hard", "the answer"))
|
||||
agent = build_e2e_agent_loop()
|
||||
|
||||
events = [event async for event in agent.act("Why?")]
|
||||
|
||||
assistant = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assert assistant and assistant[-1].content == "the answer"
|
||||
170
tests/agent_loop/e2e/test_e2e_bash.py
Normal file
170
tests/agent_loop/e2e/test_e2e_bash.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.tools.builtins.bash import BashResult
|
||||
from vibe.core.tools.permissions import RequiredPermission
|
||||
from vibe.core.types import ApprovalResponse, BaseEvent, ToolResultEvent
|
||||
|
||||
|
||||
def _bash_call(command: str, timeout: int | None = None) -> dict[str, Any]:
|
||||
arguments: dict[str, Any] = {"command": command}
|
||||
if timeout is not None:
|
||||
arguments["timeout"] = timeout
|
||||
return mistral_completion(
|
||||
"",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call_bash",
|
||||
"function": {"name": "bash", "arguments": json.dumps(arguments)},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
async def _run_bash(
|
||||
mistral_api: MistralAPI,
|
||||
command: str,
|
||||
*,
|
||||
timeout: int | None = None,
|
||||
agent_name: str = BuiltinAgentName.AUTO_APPROVE,
|
||||
approval: ApprovalResponse | None = None,
|
||||
) -> ToolResultEvent:
|
||||
mistral_api.reply(_bash_call(command, timeout), mistral_completion("done"))
|
||||
agent = build_e2e_agent_loop(
|
||||
config=e2e_config(enabled_tools=["bash"]), agent_name=agent_name
|
||||
)
|
||||
if approval is not None:
|
||||
|
||||
async def approval_callback(
|
||||
_tool_name: str,
|
||||
_args: BaseModel,
|
||||
_tool_call_id: str,
|
||||
_rp: list[RequiredPermission] | None = None,
|
||||
) -> tuple[ApprovalResponse, str | None]:
|
||||
return (approval, None)
|
||||
|
||||
agent.set_approval_callback(approval_callback)
|
||||
|
||||
events: list[BaseEvent] = [event async for event in agent.act("go")]
|
||||
return next(e for e in events if isinstance(e, ToolResultEvent))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_captures_stdout(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "echo hello")
|
||||
|
||||
bash_result = cast(BashResult, result.result)
|
||||
assert bash_result.returncode == 0
|
||||
assert "hello" in bash_result.stdout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_captures_stderr(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "echo oops >&2")
|
||||
|
||||
bash_result = cast(BashResult, result.result)
|
||||
assert "oops" in bash_result.stderr
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_nonzero_exit_surfaces_as_error(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "exit 3")
|
||||
|
||||
assert result.error is not None
|
||||
assert "Return code: 3" in result.error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_timeout_surfaces_as_error(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "sleep 5", timeout=1)
|
||||
|
||||
assert result.error is not None
|
||||
assert "timed out" in result.error.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_output_truncated_to_max_bytes(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "yes x | head -c 100000")
|
||||
|
||||
bash_result = cast(BashResult, result.result)
|
||||
assert len(bash_result.stdout) <= 16_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_denylisted_command_is_skipped(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(
|
||||
mistral_api, "vim file.txt", agent_name=BuiltinAgentName.DEFAULT
|
||||
)
|
||||
|
||||
assert result.skipped is True
|
||||
assert result.skip_reason is not None
|
||||
assert "denied" in result.skip_reason.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_allowlisted_command_runs_without_approval(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# No approval callback registered; an allowlisted command must run anyway.
|
||||
result = await _run_bash(
|
||||
mistral_api, "echo allowed", agent_name=BuiltinAgentName.DEFAULT
|
||||
)
|
||||
|
||||
assert result.skipped is False
|
||||
assert "allowed" in cast(BashResult, result.result).stdout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_non_allowlisted_command_requires_approval(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
result = await _run_bash(
|
||||
mistral_api,
|
||||
"touch newfile.txt",
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
approval=ApprovalResponse.YES,
|
||||
)
|
||||
|
||||
assert result.skipped is False
|
||||
assert (Path.cwd() / "newfile.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_non_allowlisted_command_denied_at_prompt_is_skipped(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
result = await _run_bash(
|
||||
mistral_api,
|
||||
"touch denied.txt",
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
approval=ApprovalResponse.NO,
|
||||
)
|
||||
|
||||
assert result.skipped is True
|
||||
assert not (Path.cwd() / "denied.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_command_touching_outside_workdir_requires_approval(
|
||||
mistral_api: MistralAPI, tmp_path: Path
|
||||
) -> None:
|
||||
outside = tmp_path / "outside.txt"
|
||||
result = await _run_bash(
|
||||
mistral_api,
|
||||
f"touch {outside}",
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
approval=ApprovalResponse.NO,
|
||||
)
|
||||
|
||||
assert result.skipped is True
|
||||
assert not outside.exists()
|
||||
136
tests/agent_loop/e2e/test_e2e_connectors.py
Normal file
136
tests/agent_loop/e2e/test_e2e_connectors.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from tests.constants import CONNECTORS_BOOTSTRAP_PATH
|
||||
from vibe.core.config import ConnectorConfig
|
||||
|
||||
|
||||
def _connector(
|
||||
*,
|
||||
connector_id: str = "conn-1",
|
||||
name: str = "wiki",
|
||||
is_ready: bool = True,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
auth_action: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"id": connector_id,
|
||||
"name": name,
|
||||
"status": {"is_ready": is_ready},
|
||||
"tools": tools or [],
|
||||
"auth_action": auth_action,
|
||||
}
|
||||
|
||||
|
||||
def _tool(name: str = "search", description: str = "Search docs") -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"inputSchema": {"type": "object", "properties": {"query": {"type": "string"}}},
|
||||
}
|
||||
|
||||
|
||||
def _set_bootstrap(router: respx.MockRouter, connectors: list[dict[str, Any]]) -> None:
|
||||
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
|
||||
return_value=httpx.Response(200, json={"connectors": connectors})
|
||||
)
|
||||
|
||||
|
||||
def _connectors_config(*aliases: str) -> Any:
|
||||
return e2e_config(
|
||||
connectors=[ConnectorConfig(name=alias, disabled=False) for alias in aliases]
|
||||
)
|
||||
|
||||
|
||||
async def _offered_tool_names(
|
||||
mistral_api: MistralAPI, *enabled_connectors: str
|
||||
) -> set[str]:
|
||||
# Drive one turn and read which tools the AgentLoop serialized into the
|
||||
# outgoing chat-completions request — the only connector-visible surface.
|
||||
mistral_api.reply(mistral_completion("ok"))
|
||||
agent = build_e2e_agent_loop(config=_connectors_config(*enabled_connectors))
|
||||
_ = [event async for event in agent.act("hello")]
|
||||
return {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connector_tools_are_offered_to_the_model(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(
|
||||
mock_mistral, [_connector(name="wiki", tools=[_tool("search"), _tool("read")])]
|
||||
)
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "wiki")
|
||||
|
||||
assert "connector_wiki_search" in names
|
||||
assert "connector_wiki_read" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_connector_aliases_are_disambiguated(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(
|
||||
mock_mistral,
|
||||
[
|
||||
_connector(connector_id="c-1", name="mcp", tools=[_tool("a")]),
|
||||
_connector(connector_id="c-2", name="mcp", tools=[_tool("b")]),
|
||||
],
|
||||
)
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "mcp", "mcp_2")
|
||||
|
||||
assert "connector_mcp_a" in names
|
||||
assert "connector_mcp_2_b" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_ready_connector_offers_no_tools(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(
|
||||
mock_mistral,
|
||||
[
|
||||
_connector(
|
||||
name="linear",
|
||||
is_ready=False,
|
||||
tools=[_tool("search")],
|
||||
auth_action={"type": "oauth"},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "linear")
|
||||
|
||||
assert not any(name.startswith("connector_linear") for name in names)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_connector_tools_are_withheld(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(mock_mistral, [_connector(name="wiki", tools=[_tool("search")])])
|
||||
|
||||
# No ConnectorConfig entry → the connector stays disabled by default.
|
||||
names = await _offered_tool_names(mistral_api)
|
||||
|
||||
assert "connector_wiki_search" not in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_failure_still_lets_the_agent_run(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
mock_mistral.get(CONNECTORS_BOOTSTRAP_PATH).mock(return_value=httpx.Response(500))
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "wiki")
|
||||
|
||||
assert not any(name.startswith("connector_") for name in names)
|
||||
144
tests/agent_loop/e2e/test_e2e_fork_and_plan.py
Normal file
144
tests/agent_loop/e2e/test_e2e_fork_and_plan.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.tools.builtins.ask_user_question import (
|
||||
AskUserQuestionArgs,
|
||||
AskUserQuestionResult,
|
||||
)
|
||||
from vibe.core.types import PlanReviewEndedEvent, PlanReviewRequestedEvent, Role
|
||||
from vibe.core.utils.tags import VIBE_WARNING_TAG
|
||||
|
||||
EXIT_PLAN_TOOL_CALL = [
|
||||
{
|
||||
"id": "call_exit_plan_1",
|
||||
"function": {"name": "exit_plan_mode", "arguments": "{}"},
|
||||
"index": 0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _user_message_ids(agent: Any) -> list[str]:
|
||||
return [m.message_id for m in agent.messages if m.role == Role.user]
|
||||
|
||||
|
||||
def _assistant_message_id(agent: Any) -> str:
|
||||
return next(m.message_id for m in agent.messages if m.role == Role.assistant)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_copies_all_non_system_messages(mistral_api: MistralAPI) -> None:
|
||||
# fork() with no anchor clones every non-system message into the child loop.
|
||||
mistral_api.reply(mistral_completion("Hi there!"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
forked = await agent.fork()
|
||||
|
||||
non_system = [m for m in forked.messages if m.role != Role.system]
|
||||
assert [m.role for m in non_system] == [Role.user, Role.assistant]
|
||||
assert forked.parent_session_id == agent.session_id
|
||||
assert forked.session_id != agent.session_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_from_message_id_truncates_at_next_user_turn(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# Forking from a user message keeps that turn but drops everything from the next one.
|
||||
mistral_api.reply(mistral_completion("First"), mistral_completion("Second"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Turn one")]
|
||||
_ = [event async for event in agent.act("Turn two")]
|
||||
|
||||
first_user_id = _user_message_ids(agent)[0]
|
||||
forked = await agent.fork(first_user_id)
|
||||
|
||||
contents = [m.content for m in forked.messages if m.role != Role.system]
|
||||
assert contents == ["Turn one", "First"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_from_unknown_message_id_raises(mistral_api: MistralAPI) -> None:
|
||||
# An unknown anchor id is rejected rather than silently forking everything.
|
||||
mistral_api.reply(mistral_completion("Hi"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
with pytest.raises(ValueError, match="unknown message_id"):
|
||||
await agent.fork("does-not-exist")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_from_assistant_message_id_raises(mistral_api: MistralAPI) -> None:
|
||||
# Forking is only allowed from user turns; an assistant anchor is rejected.
|
||||
mistral_api.reply(mistral_completion("Hi"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
assistant_id = _assistant_message_id(agent)
|
||||
with pytest.raises(ValueError, match="only supported for user messages"):
|
||||
await agent.fork(assistant_id)
|
||||
|
||||
|
||||
def _plan_agent(mistral_api: MistralAPI) -> Any:
|
||||
mistral_api.reply(
|
||||
mistral_completion("", tool_calls=EXIT_PLAN_TOOL_CALL),
|
||||
mistral_completion("Staying in plan mode."),
|
||||
)
|
||||
return build_e2e_agent_loop(
|
||||
config=e2e_config(enabled_tools=["exit_plan_mode"]),
|
||||
agent_name=BuiltinAgentName.PLAN,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plan_mode_emits_review_requested_and_ended_events(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# An exit_plan_mode round trip brackets the tool with review-requested/ended events.
|
||||
agent = _plan_agent(mistral_api)
|
||||
|
||||
async def stay_in_plan(_: AskUserQuestionArgs) -> AskUserQuestionResult:
|
||||
return AskUserQuestionResult(cancelled=True, answers=[])
|
||||
|
||||
agent.set_user_input_callback(stay_in_plan)
|
||||
|
||||
events = [event async for event in agent.act("Make a plan")]
|
||||
|
||||
assert any(isinstance(e, PlanReviewRequestedEvent) for e in events)
|
||||
assert any(isinstance(e, PlanReviewEndedEvent) for e in events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plan_mode_injects_updated_plan_when_file_changed(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# If the plan file changes during review, its new content is injected back as context.
|
||||
agent = _plan_agent(mistral_api)
|
||||
|
||||
plan_path: Path | None = None
|
||||
|
||||
async def edit_plan_then_decline(_: AskUserQuestionArgs) -> AskUserQuestionResult:
|
||||
assert plan_path is not None
|
||||
plan_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
plan_path.write_text("# Updated plan\nStep 1")
|
||||
return AskUserQuestionResult(cancelled=True, answers=[])
|
||||
|
||||
agent.set_user_input_callback(edit_plan_then_decline)
|
||||
|
||||
async for event in agent.act("Make a plan"):
|
||||
if isinstance(event, PlanReviewRequestedEvent):
|
||||
plan_path = event.file_path
|
||||
|
||||
injected = [m for m in agent.messages if getattr(m, "injected", False)]
|
||||
assert any(
|
||||
m.content and VIBE_WARNING_TAG in m.content and "# Updated plan" in m.content
|
||||
for m in injected
|
||||
)
|
||||
178
tests/agent_loop/e2e/test_e2e_provider_apis.py
Normal file
178
tests/agent_loop/e2e/test_e2e_provider_apis.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import (
|
||||
ProviderAPI,
|
||||
anthropic_e2e_config,
|
||||
assistant_text,
|
||||
build_e2e_agent_loop,
|
||||
openai_responses_e2e_config,
|
||||
)
|
||||
from tests.backend.data.anthropic import (
|
||||
anthropic_message,
|
||||
anthropic_reasoning_tool_use_stream,
|
||||
anthropic_request_content_blocks,
|
||||
anthropic_text_stream,
|
||||
anthropic_tool_use,
|
||||
)
|
||||
from tests.backend.data.openai_responses import (
|
||||
openai_function_call_item,
|
||||
openai_message_item,
|
||||
openai_reasoning_tool_call_stream,
|
||||
openai_response,
|
||||
openai_text_stream,
|
||||
)
|
||||
from vibe.core.types import ToolResultEvent
|
||||
|
||||
|
||||
class TestAnthropic:
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_answers(self, anthropic_api: ProviderAPI) -> None:
|
||||
# A plain prompt is answered through the Anthropic messages wire.
|
||||
anthropic_api.reply(anthropic_message("pong"))
|
||||
agent = build_e2e_agent_loop(config=anthropic_e2e_config())
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
assert agent.stats.context_tokens == 15
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams(self, anthropic_api: ProviderAPI) -> None:
|
||||
# Anthropic SSE deltas are reassembled into the final assistant content.
|
||||
anthropic_api.reply_stream(anthropic_text_stream("pong"))
|
||||
agent = build_e2e_agent_loop(
|
||||
config=anthropic_e2e_config(), enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_executes_tool_call(self, anthropic_api: ProviderAPI) -> None:
|
||||
# A tool_use turn runs the tool, then Anthropic returns the final answer.
|
||||
anthropic_api.reply(
|
||||
anthropic_tool_use("todo", {"action": "read"}),
|
||||
anthropic_message("Your list is empty."),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=anthropic_e2e_config(enabled_tools=["todo"])
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams_reasoning_and_tool_call(
|
||||
self, anthropic_api: ProviderAPI
|
||||
) -> None:
|
||||
# A streamed thinking + tool_use turn runs the tool, then replays that
|
||||
# reasoning/tool history into the follow-up Anthropic request.
|
||||
anthropic_api.reply_streams(
|
||||
anthropic_reasoning_tool_use_stream("todo", '{"action": "read"}'),
|
||||
anthropic_text_stream("Your list is empty."),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=anthropic_e2e_config(enabled_tools=["todo"]), enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
blocks = anthropic_request_content_blocks(anthropic_api.request_json)
|
||||
thinking = [b["thinking"] for b in blocks if b.get("type") == "thinking"]
|
||||
tool_uses = [b for b in blocks if b.get("type") == "tool_use"]
|
||||
|
||||
assert any("thinking..." in text for text in thinking)
|
||||
assert tool_uses
|
||||
|
||||
|
||||
class TestOpenAIResponses:
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_answers(self, openai_responses_api: ProviderAPI) -> None:
|
||||
# A plain prompt is answered through the OpenAI Responses wire.
|
||||
openai_responses_api.reply(openai_response([openai_message_item("pong")]))
|
||||
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
assert agent.stats.context_tokens == 12
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams(self, openai_responses_api: ProviderAPI) -> None:
|
||||
# OpenAI Responses SSE deltas are reassembled into the final content.
|
||||
openai_responses_api.reply_stream(openai_text_stream("pong"))
|
||||
agent = build_e2e_agent_loop(
|
||||
config=openai_responses_e2e_config(), enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_captures_reasoning(
|
||||
self, openai_responses_api: ProviderAPI
|
||||
) -> None:
|
||||
# Commentary phase output is captured as reasoning on the assistant message.
|
||||
openai_responses_api.reply(
|
||||
openai_response(
|
||||
[
|
||||
openai_message_item("Let me think.", phase="commentary"),
|
||||
openai_message_item("pong", phase="final_answer"),
|
||||
],
|
||||
output_tokens=5,
|
||||
)
|
||||
)
|
||||
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
assert any(
|
||||
m.reasoning_content and "Let me think." in m.reasoning_content
|
||||
for m in agent.messages
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_executes_tool_call(
|
||||
self, openai_responses_api: ProviderAPI
|
||||
) -> None:
|
||||
# A function_call item runs the tool, then OpenAI returns the final answer.
|
||||
openai_responses_api.reply(
|
||||
openai_response([openai_function_call_item("todo", '{"action": "read"}')]),
|
||||
openai_response([openai_message_item("Your list is empty.")]),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=openai_responses_e2e_config(enabled_tools=["todo"])
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams_reasoning_and_tool_call(
|
||||
self, openai_responses_api: ProviderAPI
|
||||
) -> None:
|
||||
# A streamed commentary + function_call turn runs the tool, then OpenAI
|
||||
# streams the final answer on the follow-up request.
|
||||
openai_responses_api.reply_streams(
|
||||
openai_reasoning_tool_call_stream("todo", '{"action": "read"}'),
|
||||
openai_text_stream("Your list is empty."),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=openai_responses_e2e_config(enabled_tools=["todo"]),
|
||||
enable_streaming=True,
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
224
tests/agent_loop/e2e/test_e2e_teleport.py
Normal file
224
tests/agent_loop/e2e/test_e2e_teleport.py
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from git import Repo
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from tests.agent_loop.e2e.conftest import build_e2e_agent_loop
|
||||
from tests.constants import (
|
||||
CONNECTORS_BOOTSTRAP_PATH,
|
||||
MISTRAL_BASE_URL,
|
||||
TELEPORT_COMPLETE_URL,
|
||||
TELEPORT_SESSIONS_PATH,
|
||||
)
|
||||
from vibe.core.agent_loop import AgentLoop, TeleportError
|
||||
from vibe.core.teleport.types import (
|
||||
TeleportCheckingGitEvent,
|
||||
TeleportCompleteEvent,
|
||||
TeleportPushingEvent,
|
||||
TeleportPushRequiredEvent,
|
||||
TeleportPushResponseEvent,
|
||||
TeleportStartingWorkflowEvent,
|
||||
)
|
||||
|
||||
# vibe_code_sessions_base_url is an exclude=True field, so it is dropped when the
|
||||
# agent manager re-derives config via model_dump(); the default url is unavoidable.
|
||||
SESSIONS_BASE_URL = "https://chat.mistral.ai"
|
||||
SESSIONS_URL = f"{SESSIONS_BASE_URL}{TELEPORT_SESSIONS_PATH}"
|
||||
|
||||
|
||||
def _sessions_ok() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"sessionId": "controller-session-id",
|
||||
"webSessionId": "web-session-id",
|
||||
"projectId": "project-id",
|
||||
"status": "running",
|
||||
"url": TELEPORT_COMPLETE_URL,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _commit(repo: Repo, message: str) -> str:
|
||||
(Path(repo.working_dir) / "file.txt").write_text(f"{message}\n")
|
||||
repo.index.add(["file.txt"])
|
||||
repo.index.commit(message)
|
||||
return repo.head.commit.hexsha
|
||||
|
||||
|
||||
def _init_repo(workdir: Path) -> Repo:
|
||||
# origin is a local bare repo so fetch/push stay offline and instant; a
|
||||
# separate github-url remote satisfies Teleport's GitHub-only detection.
|
||||
bare = Repo.init(workdir.with_name(f"{workdir.name}_origin.git"), bare=True)
|
||||
repo = Repo.init(workdir, initial_branch="work")
|
||||
repo.config_writer().set_value("user", "name", "Tester").release()
|
||||
repo.config_writer().set_value("user", "email", "t@example.com").release()
|
||||
repo.create_remote("origin", str(bare.git_dir))
|
||||
repo.create_remote("hub", "https://github.com/owner/repo.git")
|
||||
return repo
|
||||
|
||||
|
||||
def _repo_with_pushed_branch(workdir: Path) -> Repo:
|
||||
repo = _init_repo(workdir)
|
||||
_commit(repo, "initial")
|
||||
repo.git.push("origin", "work")
|
||||
return repo
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_sessions() -> Iterator[respx.MockRouter]:
|
||||
# One router stubs both hosts: the connector bootstrap (api.mistral.ai, hit on
|
||||
# agent init) and the teleport sessions endpoint (chat.mistral.ai).
|
||||
with respx.mock(assert_all_called=False) as router:
|
||||
router.get(f"{MISTRAL_BASE_URL}{CONNECTORS_BOOTSTRAP_PATH}").mock(
|
||||
return_value=httpx.Response(200, json={"connectors": []})
|
||||
)
|
||||
router.post(SESSIONS_URL).mock(return_value=_sessions_ok())
|
||||
yield router
|
||||
|
||||
|
||||
async def _drain(
|
||||
agent: AgentLoop, prompt: str | None, *, approve: bool | None = None
|
||||
) -> list[object]:
|
||||
gen = agent.teleport_to_vibe_code(prompt)
|
||||
events: list[object] = []
|
||||
response: TeleportPushResponseEvent | None = None
|
||||
while True:
|
||||
try:
|
||||
event = await gen.asend(response)
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
events.append(event)
|
||||
response = None
|
||||
if isinstance(event, TeleportPushRequiredEvent) and approve is not None:
|
||||
response = TeleportPushResponseEvent(approved=approve)
|
||||
return events
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_completes_when_branch_already_pushed(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
_repo_with_pushed_branch(tmp_working_directory)
|
||||
|
||||
events = await _drain(build_e2e_agent_loop(), "do the thing")
|
||||
|
||||
assert [type(e) for e in events] == [
|
||||
TeleportCheckingGitEvent,
|
||||
TeleportStartingWorkflowEvent,
|
||||
TeleportCompleteEvent,
|
||||
]
|
||||
assert isinstance(events[-1], TeleportCompleteEvent)
|
||||
assert events[-1].url == TELEPORT_COMPLETE_URL
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_sends_repo_metadata_and_diff(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
commit = repo.head.commit.hexsha
|
||||
(tmp_working_directory / "file.txt").write_text("uncommitted change\n")
|
||||
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
|
||||
payload = mock_sessions.calls.last.request.read().decode()
|
||||
assert "https://github.com/owner/repo.git" in payload
|
||||
assert "work" in payload
|
||||
assert commit in payload
|
||||
assert "zstd" in payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_pushes_then_completes_when_approved(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
head = _commit(repo, "second")
|
||||
|
||||
events = await _drain(build_e2e_agent_loop(), "ship it", approve=True)
|
||||
|
||||
assert [type(e) for e in events] == [
|
||||
TeleportCheckingGitEvent,
|
||||
TeleportPushRequiredEvent,
|
||||
TeleportPushingEvent,
|
||||
TeleportStartingWorkflowEvent,
|
||||
TeleportCompleteEvent,
|
||||
]
|
||||
assert repo.remote("origin").refs["work"].commit.hexsha == head
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_aborts_when_push_declined(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
_commit(repo, "second")
|
||||
|
||||
with pytest.raises(TeleportError, match="not pushed"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it", approve=False)
|
||||
|
||||
assert not mock_sessions.post(SESSIONS_URL).called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_fails_when_push_fails(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
_commit(repo, "second")
|
||||
repo.git.remote("set-url", "--push", "origin", "/nonexistent/repo.git")
|
||||
|
||||
with pytest.raises(TeleportError, match="Failed to push"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it", approve=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_requires_a_branch(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
repo.git.checkout(repo.head.commit.hexsha) # detach HEAD
|
||||
|
||||
with pytest.raises(TeleportError, match="checked-out branch"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_rejects_empty_prompt(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _init_repo(tmp_working_directory)
|
||||
_commit(repo, "initial")
|
||||
|
||||
with pytest.raises(TeleportError, match="non-empty prompt"):
|
||||
await _drain(build_e2e_agent_loop(), None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_surfaces_http_error(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
_repo_with_pushed_branch(tmp_working_directory)
|
||||
mock_sessions.post(SESSIONS_URL).mock(return_value=httpx.Response(500, text="boom"))
|
||||
|
||||
with pytest.raises(TeleportError, match="start failed"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_unsupported_without_github_remote(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = Repo.init(tmp_working_directory, initial_branch="work")
|
||||
repo.config_writer().set_value("user", "name", "Tester").release()
|
||||
repo.config_writer().set_value("user", "email", "t@example.com").release()
|
||||
_commit(repo, "initial")
|
||||
|
||||
with pytest.raises(TeleportError, match="GitHub"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
93
tests/agent_loop/e2e/test_e2e_tools.py
Normal file
93
tests/agent_loop/e2e/test_e2e_tools.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from vibe.core.tools.builtins.grep import GrepResult
|
||||
from vibe.core.tools.builtins.read import ReadResult
|
||||
from vibe.core.tools.builtins.todo import TodoResult
|
||||
from vibe.core.types import BaseEvent, ToolResultEvent
|
||||
|
||||
|
||||
def _tool_call(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
|
||||
return mistral_completion(
|
||||
"",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": f"call_{name}",
|
||||
"function": {"name": name, "arguments": json.dumps(arguments)},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
async def _run_tool(
|
||||
mistral_api: MistralAPI, name: str, arguments: dict[str, Any]
|
||||
) -> ToolResultEvent:
|
||||
# Drive one tool call against the real cwd, then a final reply, and return
|
||||
# the single tool result for the test to assert on.
|
||||
mistral_api.reply(_tool_call(name, arguments), mistral_completion("done"))
|
||||
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=[name]))
|
||||
events: list[BaseEvent] = [event async for event in agent.act("go")]
|
||||
result = next(e for e in events if isinstance(e, ToolResultEvent))
|
||||
assert result.error is None
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_file_tool_creates_file(mistral_api: MistralAPI) -> None:
|
||||
target = Path.cwd() / "note.txt"
|
||||
|
||||
await _run_tool(mistral_api, "write_file", {"path": str(target), "content": "hi\n"})
|
||||
|
||||
assert target.read_text() == "hi\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_tool_returns_file_content(mistral_api: MistralAPI) -> None:
|
||||
target = Path.cwd() / "note.txt"
|
||||
target.write_text("hello world\n")
|
||||
|
||||
result = await _run_tool(mistral_api, "read", {"file_path": str(target)})
|
||||
|
||||
assert "hello world" in cast(ReadResult, result.result).content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_tool_replaces_text(mistral_api: MistralAPI) -> None:
|
||||
target = Path.cwd() / "note.txt"
|
||||
target.write_text("hello world\n")
|
||||
|
||||
await _run_tool(
|
||||
mistral_api,
|
||||
"edit",
|
||||
{"file_path": str(target), "old_string": "hello", "new_string": "goodbye"},
|
||||
)
|
||||
|
||||
assert target.read_text() == "goodbye world\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grep_tool_finds_matches(mistral_api: MistralAPI) -> None:
|
||||
(Path.cwd() / "note.txt").write_text("needle here\n")
|
||||
|
||||
result = await _run_tool(mistral_api, "grep", {"pattern": "needle", "path": "."})
|
||||
|
||||
assert cast(GrepResult, result.result).match_count >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_todo_tool_writes_items(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_tool(
|
||||
mistral_api,
|
||||
"todo",
|
||||
{"action": "write", "todos": [{"id": "1", "content": "ship it"}]},
|
||||
)
|
||||
|
||||
assert cast(TodoResult, result.result).total_count == 1
|
||||
439
tests/agent_loop/test_agent_auto_compact.py
Normal file
439
tests/agent_loop/test_agent_auto_compact.py
Normal file
|
|
@ -0,0 +1,439 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
build_test_agent_loop,
|
||||
build_test_vibe_config,
|
||||
make_test_models,
|
||||
)
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agent_loop import CompactionFailedError
|
||||
from vibe.core.config import ModelConfig
|
||||
from vibe.core.types import (
|
||||
AssistantEvent,
|
||||
CompactEndEvent,
|
||||
CompactStartEvent,
|
||||
FunctionCall,
|
||||
LLMMessage,
|
||||
Role,
|
||||
ToolCall,
|
||||
UserMessageEvent,
|
||||
)
|
||||
|
||||
|
||||
def _get_auto_compact_properties(
|
||||
telemetry_events: list[dict[str, object]],
|
||||
) -> dict[str, object]:
|
||||
auto_compact = [
|
||||
event
|
||||
for event in telemetry_events
|
||||
if event.get("event_name") == "vibe.auto_compact_triggered"
|
||||
]
|
||||
assert len(auto_compact) == 1
|
||||
return cast(dict[str, object], auto_compact[0]["properties"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_compact_emits_correct_events(telemetry_events: list[dict]) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.stats.context_tokens = 2
|
||||
old_session_id = agent.session_id
|
||||
|
||||
events = [ev async for ev in agent.act("Hello")]
|
||||
|
||||
assert len(events) == 4
|
||||
assert isinstance(events[0], UserMessageEvent)
|
||||
assert isinstance(events[1], CompactStartEvent)
|
||||
assert isinstance(events[2], CompactEndEvent)
|
||||
assert isinstance(events[3], AssistantEvent)
|
||||
start: CompactStartEvent = events[1]
|
||||
end: CompactEndEvent = events[2]
|
||||
final: AssistantEvent = events[3]
|
||||
assert start.current_context_tokens == 2
|
||||
assert start.threshold == 1
|
||||
assert isinstance(end, CompactEndEvent)
|
||||
assert final.content == "<final>"
|
||||
|
||||
properties = _get_auto_compact_properties(telemetry_events)
|
||||
assert properties["nb_context_tokens_before"] == 2
|
||||
assert properties["auto_compact_threshold"] == 1
|
||||
assert properties["status"] == "success"
|
||||
assert properties["session_id"] == old_session_id
|
||||
assert properties["parent_session_id"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("side_effect", "expected_exception", "match", "expected_status"),
|
||||
[
|
||||
pytest.param(
|
||||
RuntimeError("boom"), RuntimeError, "boom", "failure", id="failure"
|
||||
),
|
||||
pytest.param(
|
||||
asyncio.CancelledError(),
|
||||
asyncio.CancelledError,
|
||||
None,
|
||||
"cancelled",
|
||||
id="cancelled",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_auto_compact_emits_terminal_telemetry(
|
||||
side_effect: BaseException,
|
||||
expected_exception: type[BaseException],
|
||||
match: str | None,
|
||||
expected_status: str,
|
||||
telemetry_events: list[dict],
|
||||
) -> None:
|
||||
backend = FakeBackend([[mock_llm_chunk(content="<final>")]])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.stats.context_tokens = 2
|
||||
old_session_id = agent.session_id
|
||||
|
||||
events = []
|
||||
with patch.object(agent, "compact", AsyncMock(side_effect=side_effect)):
|
||||
if match is None:
|
||||
with pytest.raises(expected_exception):
|
||||
async for event in agent.act("Hello"):
|
||||
events.append(event)
|
||||
else:
|
||||
with pytest.raises(expected_exception, match=match):
|
||||
async for event in agent.act("Hello"):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) == 2
|
||||
assert isinstance(events[0], UserMessageEvent)
|
||||
assert isinstance(events[1], CompactStartEvent)
|
||||
|
||||
properties = _get_auto_compact_properties(telemetry_events)
|
||||
assert properties["nb_context_tokens_before"] == 2
|
||||
assert properties["auto_compact_threshold"] == 1
|
||||
assert properties["status"] == expected_status
|
||||
assert properties["session_id"] == old_session_id
|
||||
assert properties["parent_session_id"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_compact_observer_sees_user_msg_not_summary() -> None:
|
||||
"""Observer sees the original user message and final response.
|
||||
|
||||
Compact internals (summary request, LLM summary) are invisible
|
||||
to the observer because they happen inside silent() / reset().
|
||||
"""
|
||||
observed: list[tuple[Role, str | None]] = []
|
||||
|
||||
def observer(msg: LLMMessage) -> None:
|
||||
observed.append((msg.role, msg.content))
|
||||
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(
|
||||
config=cfg, message_observer=observer, backend=backend
|
||||
)
|
||||
agent.stats.context_tokens = 2
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
roles = [r for r, _ in observed]
|
||||
assert roles == [Role.system, Role.user, Role.assistant]
|
||||
assert observed[1][1] == "Hello"
|
||||
assert observed[2][1] == "<final>"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_compact_observer_does_not_see_summary_request() -> None:
|
||||
"""The compact summary request and LLM response must not leak to observer."""
|
||||
observed: list[tuple[Role, str | None]] = []
|
||||
|
||||
def observer(msg: LLMMessage) -> None:
|
||||
observed.append((msg.role, msg.content))
|
||||
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(
|
||||
config=cfg, message_observer=observer, backend=backend
|
||||
)
|
||||
agent.stats.context_tokens = 2
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
contents = [c for _, c in observed]
|
||||
assert "<summary>" not in contents
|
||||
assert all("compact" not in (c or "").lower() for c in contents)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_replaces_messages_with_context() -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.stats.context_tokens = 2
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
# After compact + final response: system, compaction context, final.
|
||||
assert agent.messages[0].role == Role.system
|
||||
assert agent.messages[-1].role == Role.assistant
|
||||
assert agent.messages[-1].content == "<final>"
|
||||
|
||||
|
||||
class _ModelTrackingBackend(FakeBackend):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.requested_models: list[ModelConfig] = []
|
||||
|
||||
async def complete(self, *, model, **kwargs):
|
||||
self.requested_models.append(model)
|
||||
return await super().complete(model=model, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_uses_compaction_model() -> None:
|
||||
"""When compaction_model is set, compact() uses it instead of active_model."""
|
||||
compaction = ModelConfig(
|
||||
name="compaction-model",
|
||||
provider="mistral",
|
||||
alias="compaction",
|
||||
auto_compact_threshold=1,
|
||||
)
|
||||
backend = _ModelTrackingBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(
|
||||
models=make_test_models(auto_compact_threshold=1), compaction_model=compaction
|
||||
)
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.stats.context_tokens = 2
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert backend.requested_models[0].name == "compaction-model"
|
||||
assert backend.requested_models[1].name != "compaction-model"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_uses_active_model_when_no_compaction_model() -> None:
|
||||
"""Without compaction_model, compact() falls back to the active model."""
|
||||
backend = _ModelTrackingBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.stats.context_tokens = 2
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
active = cfg.get_active_model()
|
||||
assert backend.requested_models[0].name == active.name
|
||||
assert backend.requested_models[1].name == active.name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_appends_extra_instructions_to_prompt() -> None:
|
||||
backend = FakeBackend([[mock_llm_chunk(content="<summary>")]])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
await agent.compact(extra_instructions="focus on auth")
|
||||
|
||||
compaction_prompt = backend.requests_messages[0][-1].content
|
||||
assert compaction_prompt is not None
|
||||
assert "## Additional Instructions" in compaction_prompt
|
||||
assert "focus on auth" in compaction_prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_uses_configured_compaction_prompt(
|
||||
mock_prompts_dirs: tuple[Path, Path],
|
||||
) -> None:
|
||||
project_prompts, _ = mock_prompts_dirs
|
||||
(project_prompts / "theorem_compact.md").write_text("Summarize theorem progress")
|
||||
|
||||
backend = FakeBackend([[mock_llm_chunk(content="<summary>")]])
|
||||
cfg = build_test_vibe_config(
|
||||
models=make_test_models(auto_compact_threshold=999),
|
||||
compaction_prompt_id="theorem_compact",
|
||||
)
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
await agent.compact()
|
||||
|
||||
compaction_prompt = backend.requests_messages[0][-1].content
|
||||
assert compaction_prompt == "Summarize theorem progress"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_without_extra_instructions_has_no_additional_section() -> None:
|
||||
backend = FakeBackend([[mock_llm_chunk(content="<summary>")]])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
await agent.compact()
|
||||
|
||||
compaction_prompt = backend.requests_messages[0][-1].content
|
||||
assert compaction_prompt is not None
|
||||
assert "## Additional Instructions" not in compaction_prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_raises_on_tool_call_when_flag_enabled() -> None:
|
||||
"""With the flag on, a compaction that returns a tool call raises."""
|
||||
backend = FakeBackend([
|
||||
[
|
||||
mock_llm_chunk(
|
||||
content="",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="t1",
|
||||
index=0,
|
||||
function=FunctionCall(name="bash", arguments="{}"),
|
||||
)
|
||||
],
|
||||
)
|
||||
]
|
||||
])
|
||||
cfg = build_test_vibe_config(
|
||||
models=make_test_models(auto_compact_threshold=999),
|
||||
raise_on_compaction_failure=True,
|
||||
)
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
with pytest.raises(CompactionFailedError) as exc_info:
|
||||
await agent.compact()
|
||||
assert exc_info.value.reason == "tool_call"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_raises_on_empty_summary_when_flag_enabled() -> None:
|
||||
"""With the flag on, a compaction with empty content raises."""
|
||||
backend = FakeBackend([[mock_llm_chunk(content=" ")]])
|
||||
cfg = build_test_vibe_config(
|
||||
models=make_test_models(auto_compact_threshold=999),
|
||||
raise_on_compaction_failure=True,
|
||||
)
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
with pytest.raises(CompactionFailedError) as exc_info:
|
||||
await agent.compact()
|
||||
assert exc_info.value.reason == "empty_summary"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_falls_back_when_flag_disabled() -> None:
|
||||
"""With the flag off (default), empty content uses the legacy fallback."""
|
||||
backend = FakeBackend([[mock_llm_chunk(content="")]])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
summary = await agent.compact()
|
||||
assert summary == "(no summary available)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_message_shape_preserves_prior_user_messages() -> None:
|
||||
from vibe.core.compaction import parse_previous_user_messages
|
||||
from vibe.core.prompts import UtilityPrompt
|
||||
|
||||
summary_prefix = UtilityPrompt.COMPACT_SUMMARY_PREFIX.read()
|
||||
backend = FakeBackend([[mock_llm_chunk(content="fresh summary body")]])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
system_message_before = agent.messages[0]
|
||||
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="first real ask"))
|
||||
agent.messages.append(
|
||||
LLMMessage(role=Role.user, content="middleware ping", injected=True)
|
||||
)
|
||||
agent.messages.append(LLMMessage(role=Role.assistant, content="ack"))
|
||||
agent.messages.append(
|
||||
LLMMessage(
|
||||
role=Role.user,
|
||||
content=f"{summary_prefix}\nprior summary blob",
|
||||
injected=True,
|
||||
)
|
||||
)
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="follow-up ask"))
|
||||
agent.stats.context_tokens = 100
|
||||
|
||||
await agent.compact()
|
||||
|
||||
final = list(agent.messages)
|
||||
assert len(final) == 2 # [system, compaction_context]
|
||||
assert final[0] is system_message_before
|
||||
assert final[1].role == Role.user
|
||||
assert final[1].injected is True
|
||||
assert parse_previous_user_messages(final[1].content or "") == [
|
||||
"first real ask",
|
||||
"follow-up ask",
|
||||
]
|
||||
assert "Here are some of the most recent previous user messages" in (
|
||||
final[1].content or ""
|
||||
)
|
||||
assert "<compaction_summary>" in (final[1].content or "")
|
||||
assert "fresh summary body" in (final[1].content or "")
|
||||
# Injected and prior-summary user messages must be filtered out.
|
||||
assert all("middleware ping" not in (m.content or "") for m in final)
|
||||
assert sum("prior summary blob" in (m.content or "") for m in final) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_preserves_user_messages_across_repeated_compactions() -> None:
|
||||
from vibe.core.compaction import parse_previous_user_messages
|
||||
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="summary one")],
|
||||
[mock_llm_chunk(content="summary two")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
|
||||
agent = build_test_agent_loop(config=cfg, backend=backend)
|
||||
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="first ask"))
|
||||
agent.stats.context_tokens = 100
|
||||
await agent.compact()
|
||||
|
||||
agent.messages.append(LLMMessage(role=Role.user, content="second ask"))
|
||||
agent.stats.context_tokens = 100
|
||||
await agent.compact()
|
||||
|
||||
final = list(agent.messages)
|
||||
assert len(final) == 2
|
||||
assert parse_previous_user_messages(final[1].content or "") == [
|
||||
"first ask",
|
||||
"second ask",
|
||||
]
|
||||
83
tests/agent_loop/test_agent_auto_title.py
Normal file
83
tests/agent_loop/test_agent_auto_title.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agent_loop import AgentLoop
|
||||
from vibe.core.config import SessionLoggingConfig
|
||||
from vibe.core.types import BaseEvent, SessionTitleUpdatedEvent, UserMessageEvent
|
||||
|
||||
|
||||
def _make_agent_loop(tmp_path: Path) -> AgentLoop:
|
||||
session_logging = SessionLoggingConfig(
|
||||
save_dir=str(tmp_path / "sessions"), session_prefix="session", enabled=True
|
||||
)
|
||||
config = build_test_vibe_config(session_logging=session_logging)
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="ok")],
|
||||
[mock_llm_chunk(content="ok")],
|
||||
])
|
||||
return build_test_agent_loop(config=config, backend=backend)
|
||||
|
||||
|
||||
async def _collect(loop: AgentLoop, prompt: str, **kwargs) -> list[BaseEvent]:
|
||||
return [ev async for ev in loop.act(prompt, **kwargs)]
|
||||
|
||||
|
||||
class TestAgentLoopAutoTitleEvent:
|
||||
@pytest.mark.asyncio
|
||||
async def test_emits_event_on_first_user_message(self, tmp_path: Path) -> None:
|
||||
loop = _make_agent_loop(tmp_path)
|
||||
|
||||
events = await _collect(loop, "rendered prompt", auto_title="Pretty title")
|
||||
|
||||
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
|
||||
assert len(title_events) == 1
|
||||
assert title_events[0].title == "Pretty title"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_event_fires_after_user_message_event(self, tmp_path: Path) -> None:
|
||||
loop = _make_agent_loop(tmp_path)
|
||||
|
||||
events = await _collect(loop, "rendered", auto_title="Pretty")
|
||||
|
||||
indices = {
|
||||
type(e).__name__: i
|
||||
for i, e in enumerate(events)
|
||||
if isinstance(e, (UserMessageEvent, SessionTitleUpdatedEvent))
|
||||
}
|
||||
assert indices["UserMessageEvent"] < indices["SessionTitleUpdatedEvent"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_event_on_second_message(self, tmp_path: Path) -> None:
|
||||
loop = _make_agent_loop(tmp_path)
|
||||
await _collect(loop, "first", auto_title="First title")
|
||||
|
||||
events = await _collect(loop, "second", auto_title="Second title")
|
||||
|
||||
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
|
||||
assert title_events == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_event_when_auto_title_is_none(self, tmp_path: Path) -> None:
|
||||
loop = _make_agent_loop(tmp_path)
|
||||
|
||||
events = await _collect(loop, "rendered", auto_title=None)
|
||||
|
||||
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
|
||||
assert title_events == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_event_when_session_logging_disabled(self, tmp_path: Path) -> None:
|
||||
config = build_test_vibe_config()
|
||||
backend = FakeBackend(mock_llm_chunk(content="ok"))
|
||||
loop = build_test_agent_loop(config=config, backend=backend)
|
||||
|
||||
events = await _collect(loop, "rendered", auto_title="Pretty title")
|
||||
|
||||
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
|
||||
assert title_events == []
|
||||
505
tests/agent_loop/test_agent_backend.py
Normal file
505
tests/agent_loop/test_agent_backend.py
Normal file
|
|
@ -0,0 +1,505 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from mcp.types import (
|
||||
CreateMessageRequestParams,
|
||||
CreateMessageResult,
|
||||
SamplingMessage,
|
||||
TextContent,
|
||||
)
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
build_test_agent_loop,
|
||||
build_test_vibe_config,
|
||||
make_test_models,
|
||||
)
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
|
||||
from vibe.core.telemetry.types import EntrypointMetadata
|
||||
from vibe.core.tools.base import BaseToolConfig, ToolPermission
|
||||
from vibe.core.types import (
|
||||
Backend,
|
||||
FunctionCall,
|
||||
LLMChunk,
|
||||
LLMMessage,
|
||||
LLMUsage,
|
||||
RefusalError,
|
||||
Role,
|
||||
StopInfo,
|
||||
ToolCall,
|
||||
)
|
||||
|
||||
|
||||
def _two_model_vibe_config(active_model: str) -> VibeConfig:
|
||||
"""VibeConfig with two models so we can switch active_model."""
|
||||
models = [
|
||||
ModelConfig(
|
||||
name="mistral-vibe-cli-latest", provider="mistral", alias="devstral-latest"
|
||||
),
|
||||
ModelConfig(
|
||||
name="devstral-small-latest", provider="mistral", alias="devstral-small"
|
||||
),
|
||||
]
|
||||
providers = [
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
)
|
||||
]
|
||||
return build_test_vibe_config(
|
||||
active_model=active_model, models=models, providers=providers
|
||||
)
|
||||
|
||||
|
||||
def _make_sampling_params() -> CreateMessageRequestParams:
|
||||
return CreateMessageRequestParams(
|
||||
messages=[
|
||||
SamplingMessage(role="user", content=TextContent(type="text", text="Hi"))
|
||||
],
|
||||
systemPrompt=None,
|
||||
temperature=None,
|
||||
maxTokens=100,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passes_x_affinity_header_when_asking_an_answer(vibe_config: VibeConfig):
|
||||
backend = FakeBackend([mock_llm_chunk(content="Response")])
|
||||
agent = build_test_agent_loop(config=vibe_config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_extra_headers) > 0
|
||||
headers = backend.requests_extra_headers[0]
|
||||
assert headers is not None
|
||||
assert "x-affinity" in headers
|
||||
assert headers["x-affinity"] == agent.session_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passes_x_affinity_header_when_asking_an_answer_streaming(
|
||||
vibe_config: VibeConfig,
|
||||
):
|
||||
backend = FakeBackend([mock_llm_chunk(content="Response")])
|
||||
agent = build_test_agent_loop(
|
||||
config=vibe_config, backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_extra_headers) > 0
|
||||
headers = backend.requests_extra_headers[0]
|
||||
assert headers is not None
|
||||
assert "x-affinity" in headers
|
||||
assert headers["x-affinity"] == agent.session_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_updates_tokens_stats_based_on_backend_response(vibe_config: VibeConfig):
|
||||
chunk = mock_llm_chunk(content="Response", prompt_tokens=100, completion_tokens=50)
|
||||
backend = FakeBackend([chunk])
|
||||
agent = build_test_agent_loop(config=vibe_config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert agent.stats.context_tokens == 150
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_updates_tokens_stats_based_on_backend_response_streaming(
|
||||
vibe_config: VibeConfig,
|
||||
):
|
||||
final_chunk = mock_llm_chunk(
|
||||
content="Complete", prompt_tokens=200, completion_tokens=75
|
||||
)
|
||||
backend = FakeBackend([final_chunk])
|
||||
agent = build_test_agent_loop(
|
||||
config=vibe_config, backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert agent.stats.context_tokens == 275
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passes_session_id_to_backend(vibe_config: VibeConfig):
|
||||
backend = FakeBackend([mock_llm_chunk(content="Response")])
|
||||
agent = build_test_agent_loop(config=vibe_config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_metadata) > 0
|
||||
meta = backend.requests_metadata[0]
|
||||
assert meta is not None
|
||||
assert meta["session_id"] == agent.session_id
|
||||
assert "parent_session_id" not in meta
|
||||
assert "message_id" in meta
|
||||
assert meta["call_type"] == "main_call"
|
||||
assert meta["call_source"] == "vibe_code"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passes_parent_session_id_to_backend_after_reset(vibe_config: VibeConfig):
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="Response")],
|
||||
[mock_llm_chunk(content="Response after reset")],
|
||||
])
|
||||
agent = build_test_agent_loop(config=vibe_config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
first_session_id = agent.session_id
|
||||
|
||||
await agent._reset_session()
|
||||
[_ async for _ in agent.act("Hello again")]
|
||||
|
||||
assert len(backend.requests_metadata) >= 2
|
||||
reset_meta = backend.requests_metadata[1]
|
||||
assert reset_meta is not None
|
||||
assert reset_meta["session_id"] == agent.session_id
|
||||
assert reset_meta["parent_session_id"] == first_session_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_passes_entrypoint_metadata_to_backend(vibe_config: VibeConfig):
|
||||
metadata = EntrypointMetadata(
|
||||
agent_entrypoint="acp",
|
||||
agent_version="2.0.0",
|
||||
client_name="vibe_ide",
|
||||
client_version="0.5.0",
|
||||
)
|
||||
backend = FakeBackend([mock_llm_chunk(content="Response")])
|
||||
agent = build_test_agent_loop(
|
||||
config=vibe_config,
|
||||
backend=backend,
|
||||
enable_streaming=True,
|
||||
entrypoint_metadata=metadata,
|
||||
)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_metadata) > 0
|
||||
meta = backend.requests_metadata[0]
|
||||
assert meta is not None
|
||||
assert meta["agent_entrypoint"] == "acp"
|
||||
assert meta["agent_version"] == "2.0.0"
|
||||
assert meta["client_name"] == "vibe_ide"
|
||||
assert meta["client_version"] == "0.5.0"
|
||||
assert meta["session_id"] == agent.session_id
|
||||
assert "message_id" in meta
|
||||
assert meta["call_type"] == "main_call"
|
||||
assert meta["call_source"] == "vibe_code"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_sampling_handler_uses_updated_backend_when_agent_backend_changes():
|
||||
"""AgentLoop's MCP sampling handler uses current backend when backend is reassigned."""
|
||||
backend1 = FakeBackend([mock_llm_chunk(content="from-backend-1")])
|
||||
backend2 = FakeBackend([mock_llm_chunk(content="from-backend-2")])
|
||||
config = _two_model_vibe_config("devstral-latest")
|
||||
agent = build_test_agent_loop(config=config, backend=backend1)
|
||||
handler = agent._sampling_handler
|
||||
params = _make_sampling_params()
|
||||
context = MagicMock()
|
||||
|
||||
result1 = await handler(context, params)
|
||||
assert isinstance(result1, CreateMessageResult)
|
||||
assert result1.content.type == "text"
|
||||
assert result1.content.text == "from-backend-1"
|
||||
assert len(backend1.requests_messages) == 1
|
||||
assert len(backend2.requests_messages) == 0
|
||||
|
||||
agent.backend = backend2
|
||||
result2 = await handler(context, params)
|
||||
assert isinstance(result2, CreateMessageResult)
|
||||
assert result2.content.type == "text"
|
||||
assert result2.content.text == "from-backend-2"
|
||||
assert len(backend1.requests_messages) == 1
|
||||
assert len(backend2.requests_messages) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_sampling_handler_uses_updated_config_when_agent_config_changes():
|
||||
chunk = mock_llm_chunk(content="ok")
|
||||
backend = FakeBackend([chunk])
|
||||
config1 = _two_model_vibe_config("devstral-latest")
|
||||
config2 = _two_model_vibe_config("devstral-small")
|
||||
agent = build_test_agent_loop(config=config1, backend=backend)
|
||||
handler = agent._sampling_handler
|
||||
params = _make_sampling_params()
|
||||
context = MagicMock()
|
||||
|
||||
result1 = await handler(context, params)
|
||||
assert isinstance(result1, CreateMessageResult)
|
||||
assert result1.model == "mistral-vibe-cli-latest"
|
||||
|
||||
agent._base_config = config2
|
||||
agent.agent_manager.invalidate_config()
|
||||
result2 = await handler(context, params)
|
||||
assert isinstance(result2, CreateMessageResult)
|
||||
assert result2.model == "devstral-small-latest"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_sampling_handler_sends_secondary_call_telemetry_metadata():
|
||||
metadata = EntrypointMetadata(
|
||||
agent_entrypoint="acp",
|
||||
agent_version="2.0.0",
|
||||
client_name="vibe_ide",
|
||||
client_version="0.5.0",
|
||||
)
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="Response")],
|
||||
[mock_llm_chunk(content="Sampled response")],
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=_two_model_vibe_config("devstral-latest"),
|
||||
backend=backend,
|
||||
entrypoint_metadata=metadata,
|
||||
)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
agent.parent_session_id = "parent-session-456"
|
||||
|
||||
result = await agent._sampling_handler(MagicMock(), _make_sampling_params())
|
||||
|
||||
assert isinstance(result, CreateMessageResult)
|
||||
assert len(backend.requests_metadata) == 2
|
||||
sampling_metadata = backend.requests_metadata[1]
|
||||
assert sampling_metadata is not None
|
||||
assert sampling_metadata["agent_entrypoint"] == "acp"
|
||||
assert sampling_metadata["agent_version"] == "2.0.0"
|
||||
assert sampling_metadata["client_name"] == "vibe_ide"
|
||||
assert sampling_metadata["client_version"] == "0.5.0"
|
||||
assert sampling_metadata["session_id"] == agent.session_id
|
||||
assert sampling_metadata["parent_session_id"] == "parent-session-456"
|
||||
assert sampling_metadata["message_id"] == next(
|
||||
message.message_id for message in agent.messages if message.role == Role.user
|
||||
)
|
||||
assert sampling_metadata["call_type"] == "secondary_call"
|
||||
assert sampling_metadata["call_source"] == "vibe_code"
|
||||
|
||||
assert len(backend.requests_extra_headers) == 2
|
||||
sampling_headers = backend.requests_extra_headers[1]
|
||||
assert sampling_headers is not None
|
||||
assert sampling_headers["x-affinity"] == agent.session_id
|
||||
|
||||
|
||||
def _generic_provider_vibe_config() -> VibeConfig:
|
||||
"""VibeConfig with generic backend so no metadata header is sent."""
|
||||
providers = [
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.GENERIC,
|
||||
)
|
||||
]
|
||||
return build_test_vibe_config(providers=providers)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mistral_metadata_header_call_type_per_turn() -> None:
|
||||
"""First LLM call in a turn is main_call; second call (after tools) is secondary_call."""
|
||||
tool_call = ToolCall(
|
||||
id="call_1",
|
||||
index=0,
|
||||
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
|
||||
)
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="Checking todos.", tool_calls=[tool_call])],
|
||||
[mock_llm_chunk(content="Here are your todos.")],
|
||||
])
|
||||
config = build_test_vibe_config(
|
||||
providers=[
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
)
|
||||
],
|
||||
enabled_tools=["todo"],
|
||||
tools={"todo": BaseToolConfig(permission=ToolPermission.ALWAYS)},
|
||||
)
|
||||
agent = build_test_agent_loop(
|
||||
config=config, backend=backend, agent_name=BuiltinAgentName.AUTO_APPROVE
|
||||
)
|
||||
|
||||
[_ async for _ in agent.act("What's on my todo list?")]
|
||||
|
||||
assert len(backend.requests_metadata) == 2
|
||||
first_metadata = backend.requests_metadata[0]
|
||||
second_metadata = backend.requests_metadata[1]
|
||||
assert first_metadata is not None
|
||||
assert second_metadata is not None
|
||||
assert first_metadata["call_type"] == "main_call"
|
||||
assert second_metadata["call_type"] == "secondary_call"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_compact_emits_summary_and_next_turn_metadata() -> None:
|
||||
"""Compact emits summary then user-turn backend metadata in order."""
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
config = build_test_vibe_config(
|
||||
models=make_test_models(auto_compact_threshold=1),
|
||||
providers=[
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
)
|
||||
],
|
||||
)
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
agent.stats.context_tokens = 2
|
||||
original_session_id = agent.session_id
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_metadata) == 2
|
||||
assert len(backend.requests_extra_headers) == 2
|
||||
compact_metadata = backend.requests_metadata[0]
|
||||
user_turn_metadata = backend.requests_metadata[1]
|
||||
assert compact_metadata is not None
|
||||
assert user_turn_metadata is not None
|
||||
assert compact_metadata["call_type"] == "secondary_call"
|
||||
assert compact_metadata["session_id"] == original_session_id
|
||||
assert "parent_session_id" not in compact_metadata
|
||||
assert user_turn_metadata["call_type"] == "main_call"
|
||||
assert user_turn_metadata["session_id"] == agent.session_id
|
||||
assert user_turn_metadata["parent_session_id"] == original_session_id
|
||||
|
||||
compact_headers = backend.requests_extra_headers[0]
|
||||
user_turn_headers = backend.requests_extra_headers[1]
|
||||
assert compact_headers is not None
|
||||
assert user_turn_headers is not None
|
||||
assert compact_headers["x-affinity"] == original_session_id
|
||||
assert user_turn_headers["x-affinity"] == agent.session_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_provider_has_no_metadata_header() -> None:
|
||||
"""Non-Mistral provider does not send the metadata header."""
|
||||
backend = FakeBackend([mock_llm_chunk(content="Response")])
|
||||
config = _generic_provider_vibe_config()
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_extra_headers) == 1
|
||||
headers = backend.requests_extra_headers[0]
|
||||
assert headers is not None
|
||||
assert "metadata" not in headers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_extra_headers_are_forwarded() -> None:
|
||||
backend = FakeBackend([mock_llm_chunk(content="Response")])
|
||||
providers = [
|
||||
ProviderConfig(
|
||||
name="custom",
|
||||
api_base="https://custom.example.com/v1",
|
||||
extra_headers={"X-Custom-Auth": "token123", "X-Org-Id": "org-456"},
|
||||
)
|
||||
]
|
||||
models = [ModelConfig(name="test-model", provider="custom", alias="test")]
|
||||
config = build_test_vibe_config(
|
||||
active_model="test", models=models, providers=providers
|
||||
)
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
assert len(backend.requests_extra_headers) == 1
|
||||
headers = backend.requests_extra_headers[0]
|
||||
assert headers is not None
|
||||
assert headers["X-Custom-Auth"] == "token123"
|
||||
assert headers["X-Org-Id"] == "org-456"
|
||||
|
||||
|
||||
def _refusal_chunk() -> LLMChunk:
|
||||
return LLMChunk(
|
||||
message=LLMMessage(role=Role.assistant, content=""),
|
||||
usage=LLMUsage(prompt_tokens=10, completion_tokens=2),
|
||||
stop=StopInfo(reason="refusal"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refusal_stop_reason_raises_refusal_error(vibe_config: VibeConfig):
|
||||
backend = FakeBackend([_refusal_chunk()])
|
||||
agent = build_test_agent_loop(config=vibe_config, backend=backend)
|
||||
|
||||
with pytest.raises(RefusalError):
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refusal_stop_reason_raises_refusal_error_streaming(
|
||||
vibe_config: VibeConfig,
|
||||
):
|
||||
backend = FakeBackend([_refusal_chunk()])
|
||||
agent = build_test_agent_loop(
|
||||
config=vibe_config, backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
with pytest.raises(RefusalError):
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
|
||||
def _refusal_chunk_with_details() -> LLMChunk:
|
||||
return LLMChunk(
|
||||
message=LLMMessage(role=Role.assistant, content=""),
|
||||
usage=LLMUsage(prompt_tokens=10, completion_tokens=2),
|
||||
stop=StopInfo(
|
||||
reason="refusal",
|
||||
category="cyber",
|
||||
explanation="This request was declined for safety reasons.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refusal_error_carries_category_and_explanation(vibe_config: VibeConfig):
|
||||
backend = FakeBackend([_refusal_chunk_with_details()])
|
||||
agent = build_test_agent_loop(config=vibe_config, backend=backend)
|
||||
|
||||
with pytest.raises(RefusalError) as exc_info:
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
err = exc_info.value
|
||||
assert err.category == "cyber"
|
||||
assert err.explanation == "This request was declined for safety reasons."
|
||||
assert "This request was declined for safety reasons." in str(err)
|
||||
assert "cyber" in str(err)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refusal_error_carries_category_and_explanation_streaming(
|
||||
vibe_config: VibeConfig,
|
||||
):
|
||||
backend = FakeBackend([_refusal_chunk_with_details()])
|
||||
agent = build_test_agent_loop(
|
||||
config=vibe_config, backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
with pytest.raises(RefusalError) as exc_info:
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
|
||||
err = exc_info.value
|
||||
assert err.category == "cyber"
|
||||
assert err.explanation == "This request was declined for safety reasons."
|
||||
assert "This request was declined for safety reasons." in str(err)
|
||||
assert "cyber" in str(err)
|
||||
195
tests/agent_loop/test_agent_loop_images.py
Normal file
195
tests/agent_loop/test_agent_loop_images.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agent_loop import ImagesNotSupportedError
|
||||
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
|
||||
from vibe.core.types import Backend, FileImageSource, ImageAttachment, LLMMessage, Role
|
||||
|
||||
PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16
|
||||
|
||||
|
||||
def _config_with_vision_flag(*, supports_images: bool) -> VibeConfig:
|
||||
models = [
|
||||
ModelConfig(
|
||||
name="mistral-vibe-cli-latest",
|
||||
provider="mistral",
|
||||
alias="devstral-latest",
|
||||
supports_images=supports_images,
|
||||
)
|
||||
]
|
||||
providers = [
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
)
|
||||
]
|
||||
return build_test_vibe_config(
|
||||
active_model="devstral-latest", models=models, providers=providers
|
||||
)
|
||||
|
||||
|
||||
def _config_with_both_models() -> VibeConfig:
|
||||
models = [
|
||||
ModelConfig(
|
||||
name="vision-model",
|
||||
provider="mistral",
|
||||
alias="vision-alias",
|
||||
supports_images=True,
|
||||
),
|
||||
ModelConfig(
|
||||
name="text-model",
|
||||
provider="mistral",
|
||||
alias="text-alias",
|
||||
supports_images=False,
|
||||
),
|
||||
]
|
||||
providers = [
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
)
|
||||
]
|
||||
return build_test_vibe_config(
|
||||
active_model="vision-alias", models=models, providers=providers
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def png_attachment(tmp_path: Path) -> ImageAttachment:
|
||||
p = tmp_path / "x.png"
|
||||
p.write_bytes(PNG_BYTES)
|
||||
return ImageAttachment(
|
||||
source=FileImageSource(path=p), alias="x.png", mime_type="image/png"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_raises_when_model_lacks_vision(
|
||||
png_attachment: ImageAttachment,
|
||||
) -> None:
|
||||
config = _config_with_vision_flag(supports_images=False)
|
||||
backend = FakeBackend([mock_llm_chunk(content="ok")])
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
initial_message_count = len(agent.messages)
|
||||
|
||||
with pytest.raises(ImagesNotSupportedError):
|
||||
async for _ in agent.act("look", images=[png_attachment]):
|
||||
pass
|
||||
|
||||
assert backend.requests_extra_headers == [] # no LLM call was made
|
||||
# Capability check runs *before* checkpoint creation and history mutation,
|
||||
# so a rejected turn leaves no trace in either.
|
||||
assert agent.rewind_manager.checkpoints == []
|
||||
assert len(agent.messages) == initial_message_count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_attaches_images_to_user_message(
|
||||
png_attachment: ImageAttachment,
|
||||
) -> None:
|
||||
config = _config_with_vision_flag(supports_images=True)
|
||||
backend = FakeBackend([mock_llm_chunk(content="ok")])
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
|
||||
[_ async for _ in agent.act("look", images=[png_attachment])]
|
||||
|
||||
user_msgs = [m for m in agent.messages if m.role.value == "user"]
|
||||
assert len(user_msgs) == 1
|
||||
assert user_msgs[0].images == [png_attachment]
|
||||
|
||||
|
||||
def _seed_history_image(agent, png_attachment: ImageAttachment) -> None:
|
||||
agent.messages.append(
|
||||
LLMMessage(role=Role.user, content="earlier", images=[png_attachment])
|
||||
)
|
||||
agent.messages.append(LLMMessage(role=Role.assistant, content="seen"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enable_streaming", [False, True])
|
||||
async def test_history_images_stripped_from_backend_payload_on_non_vision_model(
|
||||
png_attachment: ImageAttachment, enable_streaming: bool
|
||||
) -> None:
|
||||
config = _config_with_both_models()
|
||||
backend = FakeBackend([mock_llm_chunk(content="ok")])
|
||||
agent = build_test_agent_loop(
|
||||
config=config, backend=backend, enable_streaming=enable_streaming
|
||||
)
|
||||
_seed_history_image(agent, png_attachment)
|
||||
agent.config.active_model = "text-alias"
|
||||
|
||||
[_ async for _ in agent.act("hi")]
|
||||
|
||||
sent_messages = backend.requests_messages[-1]
|
||||
assert all(m.images is None for m in sent_messages)
|
||||
# source-of-truth history is not mutated
|
||||
assert any(m.images == [png_attachment] for m in agent.messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("enable_streaming", [False, True])
|
||||
async def test_switch_back_to_vision_model_restores_images_in_payload(
|
||||
png_attachment: ImageAttachment, enable_streaming: bool
|
||||
) -> None:
|
||||
config = _config_with_both_models()
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="a")],
|
||||
[mock_llm_chunk(content="b")],
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=config, backend=backend, enable_streaming=enable_streaming
|
||||
)
|
||||
_seed_history_image(agent, png_attachment)
|
||||
|
||||
agent.config.active_model = "text-alias"
|
||||
[_ async for _ in agent.act("hi")]
|
||||
agent.config.active_model = "vision-alias"
|
||||
[_ async for _ in agent.act("hi again")]
|
||||
|
||||
sent_messages = backend.requests_messages[-1]
|
||||
assert any(m.images == [png_attachment] for m in sent_messages)
|
||||
|
||||
|
||||
def test_count_history_images_unsupported_by_active_model(
|
||||
png_attachment: ImageAttachment,
|
||||
) -> None:
|
||||
config = _config_with_both_models()
|
||||
agent = build_test_agent_loop(config=config)
|
||||
|
||||
assert agent.count_history_images_unsupported_by_active_model() == 0
|
||||
|
||||
_seed_history_image(agent, png_attachment)
|
||||
# active model still supports images
|
||||
assert agent.count_history_images_unsupported_by_active_model() == 0
|
||||
|
||||
agent.config.active_model = "text-alias"
|
||||
assert agent.count_history_images_unsupported_by_active_model() == 1
|
||||
|
||||
agent.messages.append(
|
||||
LLMMessage(role=Role.user, content="more", images=[png_attachment])
|
||||
)
|
||||
assert agent.count_history_images_unsupported_by_active_model() == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_images_with_non_vision_model_still_raises(
|
||||
png_attachment: ImageAttachment,
|
||||
) -> None:
|
||||
config = _config_with_both_models()
|
||||
backend = FakeBackend([mock_llm_chunk(content="ok")])
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
agent.config.active_model = "text-alias"
|
||||
|
||||
with pytest.raises(ImagesNotSupportedError):
|
||||
async for _ in agent.act("look", images=[png_attachment]):
|
||||
pass
|
||||
33
tests/agent_loop/test_agent_loop_refresh_system_prompt.py
Normal file
33
tests/agent_loop/test_agent_loop_refresh_system_prompt.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_system_prompt_preserves_scratchpad_section() -> None:
|
||||
# Regression: refresh_system_prompt must pass scratchpad_dir, otherwise
|
||||
# it silently drops the scratchpad instructions from the system prompt.
|
||||
# This fires on every session start/resume via initialize_experiments
|
||||
# and hydrate_experiments_from_session, so the LLM would lose awareness
|
||||
# of the scratchpad on the very first turn for any user with telemetry
|
||||
# enabled and a Mistral API key.
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=True,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
agent = build_test_agent_loop(config=config)
|
||||
|
||||
initial_prompt = agent.messages[0].content or ""
|
||||
assert "Scratchpad Directory" in initial_prompt
|
||||
assert agent.scratchpad_dir is not None
|
||||
assert str(agent.scratchpad_dir) in initial_prompt
|
||||
|
||||
await agent.refresh_system_prompt()
|
||||
|
||||
refreshed_prompt = agent.messages[0].content or ""
|
||||
assert "Scratchpad Directory" in refreshed_prompt
|
||||
assert str(agent.scratchpad_dir) in refreshed_prompt
|
||||
831
tests/agent_loop/test_agent_observer_streaming.py
Normal file
831
tests/agent_loop/test_agent_observer_streaming.py
Normal file
|
|
@ -0,0 +1,831 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from http import HTTPStatus
|
||||
from typing import cast
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.config import VibeConfig
|
||||
from vibe.core.llm.exceptions import BackendError, BackendErrorBuilder
|
||||
from vibe.core.middleware import (
|
||||
ConversationContext,
|
||||
MiddlewareAction,
|
||||
MiddlewareResult,
|
||||
ResetReason,
|
||||
)
|
||||
from vibe.core.tools.builtins.todo import TodoArgs
|
||||
from vibe.core.types import (
|
||||
ApprovalResponse,
|
||||
AssistantEvent,
|
||||
ContextTooLongError,
|
||||
FunctionCall,
|
||||
LLMMessage,
|
||||
RateLimitError,
|
||||
ReasoningEvent,
|
||||
Role,
|
||||
ToolCall,
|
||||
ToolCallEvent,
|
||||
ToolResultEvent,
|
||||
UserMessageEvent,
|
||||
)
|
||||
from vibe.core.utils import CancellationReason, get_user_cancellation_message
|
||||
|
||||
|
||||
class InjectBeforeMiddleware:
|
||||
injected_message = "<injected>"
|
||||
|
||||
async def before_turn(self, context: ConversationContext) -> MiddlewareResult:
|
||||
"Inject a message just before the current step executes."
|
||||
return MiddlewareResult(
|
||||
action=MiddlewareAction.INJECT_MESSAGE, message=self.injected_message
|
||||
)
|
||||
|
||||
def reset(self, reset_reason: ResetReason = ResetReason.STOP) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def make_config(
|
||||
*, enabled_tools: list[str] | None = None, tools: dict[str, dict] | None = None
|
||||
) -> VibeConfig:
|
||||
return build_test_vibe_config(
|
||||
system_prompt_id="tests",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
enabled_tools=enabled_tools or [],
|
||||
tools=tools or {},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def observer_capture() -> tuple[
|
||||
list[tuple[Role, str | None]], Callable[[LLMMessage], None]
|
||||
]:
|
||||
observed: list[tuple[Role, str | None]] = []
|
||||
|
||||
def observer(msg: LLMMessage) -> None:
|
||||
observed.append((msg.role, msg.content))
|
||||
|
||||
return observed, observer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_flushes_batched_messages_with_injection_middleware(
|
||||
observer_capture,
|
||||
) -> None:
|
||||
observed, observer = observer_capture
|
||||
|
||||
backend = FakeBackend([mock_llm_chunk(content="I can write very efficient code.")])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), message_observer=observer, backend=backend
|
||||
)
|
||||
agent.middleware_pipeline.add(InjectBeforeMiddleware())
|
||||
|
||||
async for _ in agent.act("How can you help?"):
|
||||
pass
|
||||
|
||||
assert len(observed) == 4
|
||||
assert [r for r, _ in observed] == [
|
||||
Role.system,
|
||||
Role.user,
|
||||
Role.user,
|
||||
Role.assistant,
|
||||
]
|
||||
assert observed[0][1] == "You are Vibe, a super useful programming assistant."
|
||||
assert observed[1][1] == "How can you help?"
|
||||
assert observed[2][1] == InjectBeforeMiddleware.injected_message
|
||||
assert observed[3][1] == "I can write very efficient code."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_action_flushes_user_msg_before_returning(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
|
||||
# max_turns=0 forces an immediate STOP on the first before_turn
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="My response will never reach you...")
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), message_observer=observer, max_turns=0, backend=backend
|
||||
)
|
||||
|
||||
async for _ in agent.act("Greet."):
|
||||
pass
|
||||
|
||||
assert len(observed) == 2
|
||||
# user's message should have been flushed before returning
|
||||
assert [r for r, _ in observed] == [Role.system, Role.user]
|
||||
assert observed[0][1] == "You are Vibe, a super useful programming assistant."
|
||||
assert observed[1][1] == "Greet."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_emits_user_and_assistant_msgs(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
|
||||
backend = FakeBackend([mock_llm_chunk(content="Pong!")])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), message_observer=observer, backend=backend
|
||||
)
|
||||
|
||||
async for _ in agent.act("Ping?"):
|
||||
pass
|
||||
|
||||
assert len(observed) == 3
|
||||
assert [r for r, _ in observed] == [Role.system, Role.user, Role.assistant]
|
||||
assert observed[1][1] == "Ping?"
|
||||
assert observed[2][1] == "Pong!"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_streams_chunks_in_order() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Hello"),
|
||||
mock_llm_chunk(content=" from"),
|
||||
mock_llm_chunk(content=" Vibe"),
|
||||
mock_llm_chunk(content="! "),
|
||||
mock_llm_chunk(content="More"),
|
||||
mock_llm_chunk(content=" and"),
|
||||
mock_llm_chunk(content=" end"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Stream, please.")]
|
||||
|
||||
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assert len(assistant_events) == 7
|
||||
assert [event.content for event in assistant_events] == [
|
||||
"Hello",
|
||||
" from",
|
||||
" Vibe",
|
||||
"! ",
|
||||
"More",
|
||||
" and",
|
||||
" end",
|
||||
]
|
||||
assert agent.messages[-1].role == Role.assistant
|
||||
assert agent.messages[-1].content == "Hello from Vibe! More and end"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_streaming_does_not_cleanup_tmp_files_directly() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Hello"),
|
||||
mock_llm_chunk(content=" from"),
|
||||
mock_llm_chunk(content=" Vibe"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
cleanup_spy = Mock()
|
||||
agent.session_logger.maybe_cleanup_tmp_files = cleanup_spy
|
||||
|
||||
events = [event async for event in agent.act("Stream, please.")]
|
||||
|
||||
assistant_events = [event for event in events if isinstance(event, AssistantEvent)]
|
||||
assert len(assistant_events) == 3
|
||||
assert cleanup_spy.call_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_handles_streaming_with_tool_call_events_in_sequence() -> None:
|
||||
todo_tool_call = ToolCall(
|
||||
id="tc_stream",
|
||||
index=0,
|
||||
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
|
||||
)
|
||||
backend = FakeBackend([
|
||||
[
|
||||
mock_llm_chunk(content="Checking your todos."),
|
||||
mock_llm_chunk(content="", tool_calls=[todo_tool_call]),
|
||||
],
|
||||
[mock_llm_chunk(content="Done reviewing todos.")],
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(
|
||||
enabled_tools=["todo"], tools={"todo": {"permission": "always"}}
|
||||
),
|
||||
backend=backend,
|
||||
agent_name=BuiltinAgentName.AUTO_APPROVE,
|
||||
enable_streaming=True,
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What about my todos?")]
|
||||
|
||||
assert [type(event) for event in events] == [
|
||||
UserMessageEvent,
|
||||
AssistantEvent,
|
||||
ToolCallEvent,
|
||||
ToolCallEvent,
|
||||
ToolResultEvent,
|
||||
AssistantEvent,
|
||||
]
|
||||
assert isinstance(events[0], UserMessageEvent)
|
||||
assert isinstance(events[1], AssistantEvent)
|
||||
assert events[1].content == "Checking your todos."
|
||||
assert isinstance(events[2], ToolCallEvent)
|
||||
assert events[2].args is None # streaming event
|
||||
assert events[2].tool_call_id == "tc_stream"
|
||||
assert events[2].tool_name == "todo"
|
||||
assert isinstance(events[3], ToolCallEvent)
|
||||
assert events[3].args is not None
|
||||
assert events[3].tool_name == "todo"
|
||||
assert isinstance(events[4], ToolResultEvent)
|
||||
assert events[4].error is None
|
||||
assert events[4].skipped is False
|
||||
assert isinstance(events[5], AssistantEvent)
|
||||
assert events[5].content == "Done reviewing todos."
|
||||
assert agent.messages[-1].content == "Done reviewing todos."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_handles_tool_call_chunk_with_content() -> None:
|
||||
todo_tool_call = ToolCall(
|
||||
id="tc_content",
|
||||
index=0,
|
||||
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
|
||||
)
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Preparing "),
|
||||
mock_llm_chunk(content="todo request", tool_calls=[todo_tool_call]),
|
||||
mock_llm_chunk(content=" complete"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(
|
||||
enabled_tools=["todo"], tools={"todo": {"permission": "always"}}
|
||||
),
|
||||
backend=backend,
|
||||
agent_name=BuiltinAgentName.AUTO_APPROVE,
|
||||
enable_streaming=True,
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Check todos with content.")]
|
||||
|
||||
event_types = [type(e) for e in events]
|
||||
assert Counter(event_types) == Counter({
|
||||
UserMessageEvent: 1,
|
||||
AssistantEvent: 3,
|
||||
ToolCallEvent: 2,
|
||||
ToolResultEvent: 1,
|
||||
})
|
||||
|
||||
tool_call_events = [e for e in events if isinstance(e, ToolCallEvent)]
|
||||
assert len(tool_call_events) == 2
|
||||
assert any(
|
||||
tc.args is None and tc.tool_call_id == "tc_content" for tc in tool_call_events
|
||||
)
|
||||
assert any(tc.args is not None for tc in tool_call_events)
|
||||
|
||||
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assistant_contents = {e.content for e in assistant_events}
|
||||
assert "Preparing " in assistant_contents
|
||||
assert "todo request" in assistant_contents
|
||||
assert " complete" in assistant_contents
|
||||
|
||||
assert any(
|
||||
m.role == Role.assistant and m.content == "Preparing todo request complete"
|
||||
for m in agent.messages
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_merges_streamed_tool_call_arguments() -> None:
|
||||
tool_call_part_one = ToolCall(
|
||||
id="tc_merge",
|
||||
index=0,
|
||||
function=FunctionCall(
|
||||
name="todo", arguments='{"action": "read", "note": "First '
|
||||
),
|
||||
)
|
||||
tool_call_part_two = ToolCall(
|
||||
id="tc_merge", index=0, function=FunctionCall(name="todo", arguments='part"}')
|
||||
)
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Planning: "),
|
||||
mock_llm_chunk(content="", tool_calls=[tool_call_part_one]),
|
||||
mock_llm_chunk(content="", tool_calls=[tool_call_part_two]),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(
|
||||
enabled_tools=["todo"], tools={"todo": {"permission": "always"}}
|
||||
),
|
||||
backend=backend,
|
||||
agent_name=BuiltinAgentName.AUTO_APPROVE,
|
||||
enable_streaming=True,
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Merge streamed tool call args.")]
|
||||
|
||||
assert [type(event) for event in events] == [
|
||||
UserMessageEvent,
|
||||
AssistantEvent,
|
||||
ToolCallEvent,
|
||||
ToolCallEvent,
|
||||
ToolResultEvent,
|
||||
]
|
||||
assert isinstance(events[0], UserMessageEvent)
|
||||
assert isinstance(events[2], ToolCallEvent)
|
||||
assert events[2].args is None # streaming event
|
||||
assert events[2].tool_call_id == "tc_merge"
|
||||
call_event = events[3]
|
||||
assert isinstance(call_event, ToolCallEvent)
|
||||
assert call_event.tool_call_id == "tc_merge"
|
||||
call_args = cast(TodoArgs, call_event.args)
|
||||
assert call_args.action == "read"
|
||||
assert isinstance(events[4], ToolResultEvent)
|
||||
assert events[4].error is None
|
||||
assert events[4].skipped is False
|
||||
assistant_with_calls = next(
|
||||
m for m in agent.messages if m.role == Role.assistant and m.tool_calls
|
||||
)
|
||||
reconstructed_calls = assistant_with_calls.tool_calls or []
|
||||
assert len(reconstructed_calls) == 1
|
||||
assert reconstructed_calls[0].function.arguments == (
|
||||
'{"action": "read", "note": "First part"}'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_handles_user_cancellation_during_streaming() -> None:
|
||||
class CountingMiddleware:
|
||||
def __init__(self) -> None:
|
||||
self.before_calls = 0
|
||||
|
||||
async def before_turn(self, context: ConversationContext) -> MiddlewareResult:
|
||||
self.before_calls += 1
|
||||
return MiddlewareResult()
|
||||
|
||||
def reset(self, reset_reason: ResetReason = ResetReason.STOP) -> None:
|
||||
return None
|
||||
|
||||
todo_tool_call = ToolCall(
|
||||
id="tc_cancel",
|
||||
index=0,
|
||||
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
|
||||
)
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Preparing "),
|
||||
mock_llm_chunk(content="todo request", tool_calls=[todo_tool_call]),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(
|
||||
enabled_tools=["todo"], tools={"todo": {"permission": "ask"}}
|
||||
),
|
||||
backend=backend,
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
enable_streaming=True,
|
||||
)
|
||||
middleware = CountingMiddleware()
|
||||
agent.middleware_pipeline.add(middleware)
|
||||
|
||||
async def _reject_callback(
|
||||
_name: str, _args: BaseModel, _id: str, _rp: list | None = None
|
||||
) -> tuple[ApprovalResponse, str | None]:
|
||||
return (
|
||||
ApprovalResponse.NO,
|
||||
str(get_user_cancellation_message(CancellationReason.OPERATION_CANCELLED)),
|
||||
)
|
||||
|
||||
agent.set_approval_callback(_reject_callback)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
events = [event async for event in agent.act("Cancel mid stream?")]
|
||||
|
||||
assert [type(event) for event in events] == [
|
||||
UserMessageEvent,
|
||||
AssistantEvent,
|
||||
ToolCallEvent,
|
||||
AssistantEvent,
|
||||
ToolCallEvent,
|
||||
ToolResultEvent,
|
||||
]
|
||||
assert middleware.before_calls == 1
|
||||
assert isinstance(events[-1], ToolResultEvent)
|
||||
assert events[-1].skipped is True
|
||||
assert events[-1].skip_reason is not None
|
||||
assert "<user_cancellation>" in events[-1].skip_reason
|
||||
assert events[-1].cancelled is True
|
||||
assert agent.session_logger.save_interaction.await_count >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_flushes_and_logs_when_streaming_errors(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend = FakeBackend(exception_to_raise=RuntimeError("boom in streaming"))
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=True,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom in streaming"):
|
||||
[_ async for _ in agent.act("Trigger stream failure")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limit(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
response = httpx.Response(
|
||||
HTTPStatus.TOO_MANY_REQUESTS, request=httpx.Request("POST", "http://test")
|
||||
)
|
||||
error = httpx.HTTPStatusError(
|
||||
"rate limited", request=response.request, response=response
|
||||
)
|
||||
backend_error = BackendErrorBuilder.build_http_error(
|
||||
provider="mistral",
|
||||
endpoint="test",
|
||||
error=error,
|
||||
model="test-model",
|
||||
messages=[],
|
||||
temperature=0.0,
|
||||
has_tools=False,
|
||||
tool_choice=None,
|
||||
)
|
||||
backend = FakeBackend(exception_to_raise=backend_error)
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=True,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(RateLimitError):
|
||||
[_ async for _ in agent.act("Trigger rate limit failure while streaming")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
def _build_context_too_long_backend_error() -> BackendError:
|
||||
response = httpx.Response(
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
request=httpx.Request("POST", "http://test"),
|
||||
text='{"message": "Context too long"}',
|
||||
)
|
||||
error = httpx.HTTPStatusError(
|
||||
"context too long", request=response.request, response=response
|
||||
)
|
||||
return BackendErrorBuilder.build_http_error(
|
||||
provider="mistral",
|
||||
endpoint="test",
|
||||
error=error,
|
||||
model="test-model",
|
||||
messages=[],
|
||||
temperature=0.0,
|
||||
has_tools=False,
|
||||
tool_choice=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_too_long_streaming(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend_error = _build_context_too_long_backend_error()
|
||||
backend = FakeBackend(exception_to_raise=backend_error)
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=True,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(ContextTooLongError):
|
||||
[_ async for _ in agent.act("Trigger context too long while streaming")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_too_long_non_streaming(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend_error = _build_context_too_long_backend_error()
|
||||
backend = FakeBackend(exception_to_raise=backend_error)
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=False,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(ContextTooLongError):
|
||||
[_ async for _ in agent.act("Trigger context too long without streaming")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
class _NonRetryableError(Exception):
|
||||
# Mimics Temporal's ``ApplicationError(non_retryable=True)`` without
|
||||
# pulling temporalio into vibe's test deps. The wrap-site check relies
|
||||
# on the truthy ``non_retryable`` attribute, not the concrete type.
|
||||
non_retryable = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_retryable_passes_through_streaming(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend = FakeBackend(exception_to_raise=_NonRetryableError("auth failed"))
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=True,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(_NonRetryableError, match="auth failed"):
|
||||
[_ async for _ in agent.act("Trigger non-retryable failure while streaming")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_retryable_passes_through_non_streaming(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend = FakeBackend(exception_to_raise=_NonRetryableError("auth failed"))
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=False,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(_NonRetryableError, match="auth failed"):
|
||||
[_ async for _ in agent.act("Trigger non-retryable failure without streaming")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
def _wrap_with_cause(message: str, cause: BaseException) -> RuntimeError:
|
||||
# Mirrors how Temporal raises ``ActivityError`` on the workflow side with
|
||||
# the original ``ApplicationError`` chained as ``__cause__`` — except we
|
||||
# don't import temporalio. The wrap-site check has to traverse the chain
|
||||
# to find ``non_retryable`` regardless of how deep it is.
|
||||
error = RuntimeError(message)
|
||||
error.__cause__ = cause
|
||||
return error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_retryable_via_cause_chain_streaming(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
wrapped = _wrap_with_cause(
|
||||
"Activity task failed", _NonRetryableError("auth failed")
|
||||
)
|
||||
backend = FakeBackend(exception_to_raise=wrapped)
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=True,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Activity task failed"):
|
||||
[_ async for _ in agent.act("Trigger non-retryable via cause chain")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_retryable_via_cause_chain_non_streaming(observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
wrapped = _wrap_with_cause(
|
||||
"Activity task failed", _NonRetryableError("auth failed")
|
||||
)
|
||||
backend = FakeBackend(exception_to_raise=wrapped)
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(),
|
||||
backend=backend,
|
||||
message_observer=observer,
|
||||
enable_streaming=False,
|
||||
)
|
||||
agent.session_logger.save_interaction = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Activity task failed"):
|
||||
[_ async for _ in agent.act("Trigger non-retryable via cause chain")]
|
||||
|
||||
assert [role for role, _ in observed] == [Role.system, Role.user]
|
||||
assert agent.session_logger.save_interaction.await_count == 1
|
||||
|
||||
|
||||
def _snapshot_events(events: list) -> list[tuple[str, str]]:
|
||||
return [
|
||||
(type(e).__name__, e.content)
|
||||
for e in events
|
||||
if isinstance(e, (AssistantEvent, ReasoningEvent))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_yields_before_content_on_transition() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="", reasoning_content="Let me think"),
|
||||
mock_llm_chunk(content="", reasoning_content=" about this"),
|
||||
mock_llm_chunk(content="", reasoning_content=" problem..."),
|
||||
mock_llm_chunk(content="The answer is 42."),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's the answer?")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("ReasoningEvent", "Let me think"),
|
||||
("ReasoningEvent", " about this"),
|
||||
("ReasoningEvent", " problem..."),
|
||||
("AssistantEvent", "The answer is 42."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_yields_per_chunk() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="", reasoning_content="Step 1"),
|
||||
mock_llm_chunk(content="", reasoning_content=", Step 2"),
|
||||
mock_llm_chunk(content="", reasoning_content=", Step 3"),
|
||||
mock_llm_chunk(content="", reasoning_content=", Step 4"),
|
||||
mock_llm_chunk(content="", reasoning_content=", Step 5"),
|
||||
mock_llm_chunk(content="", reasoning_content=", Step 6"),
|
||||
mock_llm_chunk(content="", reasoning_content=", Final"),
|
||||
mock_llm_chunk(content="Done thinking!"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Think step by step")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("ReasoningEvent", "Step 1"),
|
||||
("ReasoningEvent", ", Step 2"),
|
||||
("ReasoningEvent", ", Step 3"),
|
||||
("ReasoningEvent", ", Step 4"),
|
||||
("ReasoningEvent", ", Step 5"),
|
||||
("ReasoningEvent", ", Step 6"),
|
||||
("ReasoningEvent", ", Final"),
|
||||
("AssistantEvent", "Done thinking!"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_yields_before_reasoning_on_transition() -> None:
|
||||
"""When content chunks arrive and reasoning arrives, content yields first."""
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Starting the response"),
|
||||
mock_llm_chunk(content=" here..."),
|
||||
mock_llm_chunk(content="", reasoning_content="Wait, let me reconsider"),
|
||||
mock_llm_chunk(content="", reasoning_content=" this approach..."),
|
||||
mock_llm_chunk(content="Actually, the final answer."),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Give me an answer")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("AssistantEvent", "Starting the response"),
|
||||
("AssistantEvent", " here..."),
|
||||
("ReasoningEvent", "Wait, let me reconsider"),
|
||||
("ReasoningEvent", " this approach..."),
|
||||
("AssistantEvent", "Actually, the final answer."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interleaved_reasoning_content_preserves_order() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="", reasoning_content="Think 1"),
|
||||
mock_llm_chunk(content="Answer 1 "),
|
||||
mock_llm_chunk(content="", reasoning_content="Think 2"),
|
||||
mock_llm_chunk(content="Answer 2 "),
|
||||
mock_llm_chunk(content="", reasoning_content="Think 3"),
|
||||
mock_llm_chunk(content="Answer 3"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Interleaved test")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("ReasoningEvent", "Think 1"),
|
||||
("AssistantEvent", "Answer 1 "),
|
||||
("ReasoningEvent", "Think 2"),
|
||||
("AssistantEvent", "Answer 2 "),
|
||||
("ReasoningEvent", "Think 3"),
|
||||
("AssistantEvent", "Answer 3"),
|
||||
]
|
||||
|
||||
assistant_msg = next(m for m in agent.messages if m.role == Role.assistant)
|
||||
assert assistant_msg.reasoning_content == "Think 1Think 2Think 3"
|
||||
assert assistant_msg.content == "Answer 1 Answer 2 Answer 3"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_reasoning_chunks_yields_reasoning_event() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="", reasoning_content="Just thinking..."),
|
||||
mock_llm_chunk(content="", reasoning_content=" nothing to say yet."),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Silent thinking")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("ReasoningEvent", "Just thinking..."),
|
||||
("ReasoningEvent", " nothing to say yet."),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_buffers_flush_in_correct_order() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="", reasoning_content="Final thought"),
|
||||
mock_llm_chunk(content="Final words"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("End buffers test")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("ReasoningEvent", "Final thought"),
|
||||
("AssistantEvent", "Final words"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_content_chunks_do_not_trigger_false_yields() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="", reasoning_content="Reasoning here"),
|
||||
mock_llm_chunk(content=""), # Empty content shouldn't yield
|
||||
mock_llm_chunk(content="", reasoning_content=" more reasoning"),
|
||||
mock_llm_chunk(content="Actual content"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Empty content test")]
|
||||
|
||||
assert _snapshot_events(events) == [
|
||||
("ReasoningEvent", "Reasoning here"),
|
||||
("ReasoningEvent", " more reasoning"),
|
||||
("AssistantEvent", "Actual content"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_assistant_event_message_id_matches_stored_message() -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(content="Hello"),
|
||||
mock_llm_chunk(content=" world"),
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), backend=backend, enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Test")]
|
||||
|
||||
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assert len(assistant_events) == 2
|
||||
|
||||
# All chunks of the same assistant turn share one message_id
|
||||
message_ids = {e.message_id for e in assistant_events}
|
||||
assert len(message_ids) == 1
|
||||
|
||||
# The stored LLMMessage must carry that same message_id
|
||||
stored_msg = next(m for m in agent.messages if m.role == Role.assistant)
|
||||
assert stored_msg.message_id == assistant_events[0].message_id
|
||||
109
tests/agent_loop/test_agent_override_resolve_permission.py
Normal file
109
tests/agent_loop/test_agent_override_resolve_permission.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.paths import PLANS_DIR
|
||||
from vibe.core.tools.base import ToolPermission
|
||||
|
||||
|
||||
class TestPlanAgentWriteFileResolvePermission:
|
||||
"""Plan agent sets write_file to NEVER with allowlist=[plans/*].
|
||||
resolve_permission must use this, not the base config.
|
||||
"""
|
||||
|
||||
def test_write_file_to_non_plan_path_denied_in_plan_mode(self) -> None:
|
||||
config = build_test_vibe_config()
|
||||
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
|
||||
|
||||
tool = agent.tool_manager.get("write_file")
|
||||
from vibe.core.tools.builtins.write_file import WriteFileArgs
|
||||
|
||||
args = WriteFileArgs(path="/some/random/file.py", content="hello")
|
||||
|
||||
ctx = tool.resolve_permission(args)
|
||||
|
||||
# With plan agent override: permission should be NEVER
|
||||
# (unless the path matches the plans allowlist)
|
||||
assert ctx is not None
|
||||
assert ctx.permission == ToolPermission.NEVER
|
||||
|
||||
def test_write_file_to_plan_path_allowed_in_plan_mode(self) -> None:
|
||||
config = build_test_vibe_config()
|
||||
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
|
||||
|
||||
tool = agent.tool_manager.get("write_file")
|
||||
from vibe.core.tools.builtins.write_file import WriteFileArgs
|
||||
|
||||
plan_path = str(PLANS_DIR.path / "my-plan.md")
|
||||
args = WriteFileArgs(path=plan_path, content="# Plan")
|
||||
|
||||
ctx = tool.resolve_permission(args)
|
||||
|
||||
# Plan path is in the allowlist, so should be ALWAYS
|
||||
assert ctx is not None
|
||||
assert ctx.permission == ToolPermission.ALWAYS
|
||||
|
||||
def test_edit_to_non_plan_path_denied_in_plan_mode(self) -> None:
|
||||
config = build_test_vibe_config()
|
||||
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
|
||||
|
||||
tool = agent.tool_manager.get("edit")
|
||||
from vibe.core.tools.builtins.edit import EditArgs
|
||||
|
||||
args = EditArgs(file_path="/some/file.py", old_string="a", new_string="b")
|
||||
|
||||
ctx = tool.resolve_permission(args)
|
||||
|
||||
assert ctx is not None
|
||||
assert ctx.permission == ToolPermission.NEVER
|
||||
|
||||
|
||||
class TestAcceptEditsAgentResolvePermission:
|
||||
"""Accept-edits agent sets write_file/edit to ALWAYS.
|
||||
resolve_permission must reflect this.
|
||||
"""
|
||||
|
||||
def test_write_file_always_in_accept_edits_mode(self) -> None:
|
||||
config = build_test_vibe_config()
|
||||
agent = build_test_agent_loop(
|
||||
config=config, agent_name=BuiltinAgentName.ACCEPT_EDITS
|
||||
)
|
||||
|
||||
tool = agent.tool_manager.get("write_file")
|
||||
from vibe.core.tools.builtins.write_file import WriteFileArgs
|
||||
|
||||
# Use a workdir-relative path; outside-workdir always requires ASK
|
||||
# regardless of agent permission.
|
||||
args = WriteFileArgs(path="file.py", content="hello")
|
||||
|
||||
ctx = tool.resolve_permission(args)
|
||||
|
||||
# Inside workdir, no allowlist/denylist/sensitive match → None,
|
||||
# so the caller falls through to config permission (ALWAYS).
|
||||
assert ctx is None
|
||||
|
||||
|
||||
class TestAgentOverrideNotLeakedAcrossSwitches:
|
||||
"""Switching agents must change what resolve_permission returns."""
|
||||
|
||||
def test_switch_from_plan_to_default_restores_write_permission(self) -> None:
|
||||
config = build_test_vibe_config()
|
||||
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
|
||||
|
||||
tool = agent.tool_manager.get("write_file")
|
||||
from vibe.core.tools.builtins.write_file import WriteFileArgs
|
||||
|
||||
args = WriteFileArgs(path="/some/file.py", content="hello")
|
||||
|
||||
# In plan mode: should be NEVER
|
||||
ctx_plan = tool.resolve_permission(args)
|
||||
assert ctx_plan is not None
|
||||
assert ctx_plan.permission == ToolPermission.NEVER
|
||||
|
||||
# Switch to default
|
||||
agent.agent_manager.switch_profile(BuiltinAgentName.DEFAULT)
|
||||
|
||||
# In default mode: should NOT be NEVER
|
||||
ctx_default = tool.resolve_permission(args)
|
||||
assert ctx_default is not None
|
||||
assert ctx_default.permission != ToolPermission.NEVER
|
||||
752
tests/agent_loop/test_agent_stats.py
Normal file
752
tests/agent_loop/test_agent_stats.py
Normal file
|
|
@ -0,0 +1,752 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import (
|
||||
build_test_agent_loop,
|
||||
build_test_vibe_config,
|
||||
make_test_models,
|
||||
)
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.config import (
|
||||
ModelConfig,
|
||||
ProviderConfig,
|
||||
SessionLoggingConfig,
|
||||
VibeConfig,
|
||||
)
|
||||
from vibe.core.tools.base import ToolPermission
|
||||
from vibe.core.types import (
|
||||
AgentStats,
|
||||
AssistantEvent,
|
||||
Backend,
|
||||
CompactEndEvent,
|
||||
CompactStartEvent,
|
||||
FunctionCall,
|
||||
LLMMessage,
|
||||
Role,
|
||||
ToolCall,
|
||||
UserMessageEvent,
|
||||
)
|
||||
|
||||
|
||||
def make_config(
|
||||
*,
|
||||
system_prompt_id: str = "tests",
|
||||
active_model: str = "devstral-latest",
|
||||
input_price: float = 0.4,
|
||||
output_price: float = 2.0,
|
||||
disable_logging: bool = True,
|
||||
auto_compact_threshold: int = 0,
|
||||
include_project_context: bool = False,
|
||||
include_prompt_detail: bool = False,
|
||||
enabled_tools: list[str] | None = None,
|
||||
todo_permission: ToolPermission = ToolPermission.ALWAYS,
|
||||
) -> VibeConfig:
|
||||
models = [
|
||||
ModelConfig(
|
||||
name="mistral-vibe-cli-latest",
|
||||
provider="mistral",
|
||||
alias="devstral-latest",
|
||||
input_price=input_price,
|
||||
output_price=output_price,
|
||||
auto_compact_threshold=auto_compact_threshold,
|
||||
),
|
||||
ModelConfig(
|
||||
name="devstral-small-latest",
|
||||
provider="mistral",
|
||||
alias="devstral-small",
|
||||
input_price=0.1,
|
||||
output_price=0.3,
|
||||
auto_compact_threshold=auto_compact_threshold,
|
||||
),
|
||||
ModelConfig(
|
||||
name="strawberry",
|
||||
provider="lechat",
|
||||
alias="strawberry",
|
||||
input_price=2.5,
|
||||
output_price=10.0,
|
||||
auto_compact_threshold=auto_compact_threshold,
|
||||
),
|
||||
]
|
||||
providers = [
|
||||
ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="MISTRAL_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
),
|
||||
ProviderConfig(
|
||||
name="lechat",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="LECHAT_API_KEY",
|
||||
backend=Backend.MISTRAL,
|
||||
),
|
||||
]
|
||||
return build_test_vibe_config(
|
||||
session_logging=SessionLoggingConfig(enabled=not disable_logging),
|
||||
system_prompt_id=system_prompt_id,
|
||||
include_project_context=include_project_context,
|
||||
include_prompt_detail=include_prompt_detail,
|
||||
active_model=active_model,
|
||||
models=models,
|
||||
providers=providers,
|
||||
enabled_tools=enabled_tools or [],
|
||||
tools={"todo": {"permission": todo_permission.value}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def observer_capture() -> tuple[list[LLMMessage], Callable[[LLMMessage], None]]:
|
||||
observed: list[LLMMessage] = []
|
||||
|
||||
def observer(msg: LLMMessage) -> None:
|
||||
observed.append(msg)
|
||||
|
||||
return observed, observer
|
||||
|
||||
|
||||
class TestAgentStatsHelpers:
|
||||
def test_update_pricing(self) -> None:
|
||||
stats = AgentStats()
|
||||
stats.update_pricing(1.5, 3.0)
|
||||
assert stats.input_price_per_million == 1.5
|
||||
assert stats.output_price_per_million == 3.0
|
||||
|
||||
def test_reset_context_state_preserves_cumulative(self) -> None:
|
||||
stats = AgentStats(
|
||||
steps=5,
|
||||
session_prompt_tokens=1000,
|
||||
session_completion_tokens=500,
|
||||
tool_calls_succeeded=3,
|
||||
tool_calls_failed=1,
|
||||
context_tokens=800,
|
||||
last_turn_prompt_tokens=100,
|
||||
last_turn_completion_tokens=50,
|
||||
last_turn_duration=1.5,
|
||||
tokens_per_second=33.3,
|
||||
input_price_per_million=0.4,
|
||||
output_price_per_million=2.0,
|
||||
)
|
||||
|
||||
stats.reset_context_state()
|
||||
|
||||
assert stats.steps == 5
|
||||
assert stats.session_prompt_tokens == 1000
|
||||
assert stats.session_completion_tokens == 500
|
||||
assert stats.tool_calls_succeeded == 3
|
||||
assert stats.tool_calls_failed == 1
|
||||
assert stats.input_price_per_million == 0.4
|
||||
assert stats.output_price_per_million == 2.0
|
||||
|
||||
assert stats.context_tokens == 0
|
||||
assert stats.last_turn_prompt_tokens == 0
|
||||
assert stats.last_turn_completion_tokens == 0
|
||||
assert stats.last_turn_duration == 0.0
|
||||
assert stats.tokens_per_second == 0.0
|
||||
|
||||
def test_session_cost_computed_from_current_pricing(self) -> None:
|
||||
stats = AgentStats(
|
||||
session_prompt_tokens=1_000_000,
|
||||
session_completion_tokens=500_000,
|
||||
input_price_per_million=1.0,
|
||||
output_price_per_million=2.0,
|
||||
)
|
||||
# Cost = 1M * $1/M + 0.5M * $2/M = $1 + $1 = $2
|
||||
assert stats.session_cost == 2.0
|
||||
|
||||
stats.update_pricing(2.0, 4.0)
|
||||
# Cost = 1M * $2/M + 0.5M * $4/M = $2 + $2 = $4
|
||||
assert stats.session_cost == 4.0
|
||||
|
||||
|
||||
class TestReloadPreservesStats:
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_session_tokens(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="First response"))
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
old_session_prompt = agent.stats.session_prompt_tokens
|
||||
old_session_completion = agent.stats.session_completion_tokens
|
||||
assert old_session_prompt > 0
|
||||
assert old_session_completion > 0
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert agent.stats.session_prompt_tokens == old_session_prompt
|
||||
assert agent.stats.session_completion_tokens == old_session_completion
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_tool_call_stats(self) -> None:
|
||||
backend = FakeBackend([
|
||||
mock_llm_chunk(
|
||||
content="Calling tool",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="tc1",
|
||||
index=0,
|
||||
function=FunctionCall(
|
||||
name="todo", arguments='{"action": "read"}'
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
mock_llm_chunk(content="Done"),
|
||||
])
|
||||
config = make_config(enabled_tools=["todo"])
|
||||
agent = build_test_agent_loop(
|
||||
config=config, agent_name=BuiltinAgentName.AUTO_APPROVE, backend=backend
|
||||
)
|
||||
|
||||
async for _ in agent.act("Check todos"):
|
||||
pass
|
||||
|
||||
assert agent.stats.tool_calls_succeeded == 1
|
||||
assert agent.stats.tool_calls_agreed == 1
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert agent.stats.tool_calls_succeeded == 1
|
||||
assert agent.stats.tool_calls_agreed == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_steps(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="R1")],
|
||||
[mock_llm_chunk(content="R2")],
|
||||
])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("First"):
|
||||
pass
|
||||
async for _ in agent.act("Second"):
|
||||
pass
|
||||
|
||||
old_steps = agent.stats.steps
|
||||
assert old_steps >= 2
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert agent.stats.steps == old_steps
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_context_tokens_when_messages_preserved(
|
||||
self,
|
||||
) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
assert agent.stats.context_tokens > 0
|
||||
initial_context_tokens = agent.stats.context_tokens
|
||||
assert len(agent.messages) > 1
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert len(agent.messages) > 1
|
||||
assert agent.stats.context_tokens == initial_context_tokens
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_resets_context_tokens_when_no_messages(self) -> None:
|
||||
backend = FakeBackend([])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
assert len(agent.messages) == 1
|
||||
assert agent.stats.context_tokens == 0
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert len(agent.messages) == 1
|
||||
assert agent.stats.context_tokens == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_resets_context_tokens_when_system_prompt_changes(
|
||||
self,
|
||||
) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config1 = make_config(system_prompt_id="tests")
|
||||
config2 = make_config(system_prompt_id="cli")
|
||||
agent = build_test_agent_loop(config=config1, backend=backend)
|
||||
[_ async for _ in agent.act("Hello")]
|
||||
original_context_tokens = agent.stats.context_tokens
|
||||
assert original_context_tokens > 0
|
||||
assert len(agent.messages) > 1
|
||||
|
||||
await agent.reload_with_initial_messages(base_config=config2)
|
||||
|
||||
assert len(agent.messages) > 1
|
||||
assert agent.stats.context_tokens == original_context_tokens
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_updates_pricing_from_new_model(self, monkeypatch) -> None:
|
||||
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
|
||||
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config_mistral = make_config(active_model="devstral-latest")
|
||||
agent = build_test_agent_loop(config=config_mistral, backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
assert agent.stats.input_price_per_million == 0.4
|
||||
assert agent.stats.output_price_per_million == 2.0
|
||||
|
||||
config_other = make_config(active_model="strawberry")
|
||||
await agent.reload_with_initial_messages(base_config=config_other)
|
||||
|
||||
assert agent.stats.input_price_per_million == 2.5
|
||||
assert agent.stats.output_price_per_million == 10.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_accumulates_tokens_across_configs(self, monkeypatch) -> None:
|
||||
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
|
||||
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="First")],
|
||||
[mock_llm_chunk(content="After reload")],
|
||||
])
|
||||
config1 = make_config(active_model="devstral-latest")
|
||||
agent = build_test_agent_loop(config=config1, backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
tokens_after_first = (
|
||||
agent.stats.session_prompt_tokens + agent.stats.session_completion_tokens
|
||||
)
|
||||
|
||||
config2 = make_config(active_model="strawberry")
|
||||
await agent.reload_with_initial_messages(base_config=config2)
|
||||
|
||||
async for _ in agent.act("Continue"):
|
||||
pass
|
||||
|
||||
tokens_after_second = (
|
||||
agent.stats.session_prompt_tokens + agent.stats.session_completion_tokens
|
||||
)
|
||||
assert tokens_after_second > tokens_after_first
|
||||
|
||||
|
||||
class TestReloadPreservesMessages:
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_preserves_conversation_messages(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
assert len(agent.messages) == 3
|
||||
old_user_content = agent.messages[1].content
|
||||
old_assistant_content = agent.messages[2].content
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert len(agent.messages) == 3
|
||||
assert agent.messages[0].role == Role.system
|
||||
assert agent.messages[1].role == Role.user
|
||||
assert agent.messages[1].content == old_user_content
|
||||
assert agent.messages[2].role == Role.assistant
|
||||
assert agent.messages[2].content == old_assistant_content
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_updates_system_prompt_preserves_rest(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config1 = make_config(system_prompt_id="tests")
|
||||
agent = build_test_agent_loop(config=config1, backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
old_system = agent.messages[0].content
|
||||
old_user = agent.messages[1].content
|
||||
|
||||
config2 = make_config(system_prompt_id="cli")
|
||||
await agent.reload_with_initial_messages(base_config=config2)
|
||||
|
||||
assert agent.messages[0].content != old_system
|
||||
assert agent.messages[1].content == old_user
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_with_no_messages_stays_empty(self) -> None:
|
||||
backend = FakeBackend([])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
assert len(agent.messages) == 1
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert len(agent.messages) == 1
|
||||
assert agent.messages[0].role == Role.system
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_does_not_reemit_to_observer(self, observer_capture) -> None:
|
||||
observed, observer = observer_capture
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), message_observer=observer, backend=backend
|
||||
)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
observed.clear()
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert len(observed) == 0
|
||||
|
||||
|
||||
class TestCompactStatsHandling:
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_preserves_cumulative_stats(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="First response")],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Build something"):
|
||||
pass
|
||||
|
||||
tokens_before_compact = agent.stats.session_prompt_tokens
|
||||
completions_before = agent.stats.session_completion_tokens
|
||||
steps_before = agent.stats.steps
|
||||
|
||||
await agent.compact()
|
||||
|
||||
# Cumulative stats include the compact turn
|
||||
assert agent.stats.session_prompt_tokens > tokens_before_compact
|
||||
assert agent.stats.session_completion_tokens > completions_before
|
||||
assert agent.stats.steps > steps_before
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_updates_context_tokens(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="Long response " * 100)],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Do something complex"):
|
||||
pass
|
||||
|
||||
context_before = agent.stats.context_tokens
|
||||
|
||||
await agent.compact()
|
||||
|
||||
assert agent.stats.context_tokens < context_before
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_preserves_tool_call_stats(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[
|
||||
mock_llm_chunk(
|
||||
content="Using tool",
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id="tc1",
|
||||
index=0,
|
||||
function=FunctionCall(
|
||||
name="todo", arguments='{"action": "read"}'
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
mock_llm_chunk(content=" todo"),
|
||||
],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
config = make_config(enabled_tools=["todo"])
|
||||
agent = build_test_agent_loop(
|
||||
config=config, agent_name=BuiltinAgentName.AUTO_APPROVE, backend=backend
|
||||
)
|
||||
|
||||
async for _ in agent.act("Check todos"):
|
||||
pass
|
||||
|
||||
assert agent.stats.tool_calls_succeeded == 1
|
||||
|
||||
await agent.compact()
|
||||
|
||||
assert agent.stats.tool_calls_succeeded == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_resets_session_id(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="Long response " * 100)],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(disable_logging=False), backend=backend
|
||||
)
|
||||
|
||||
original_session_id = agent.session_id
|
||||
original_logger_session_id = agent.session_logger.session_id
|
||||
|
||||
assert agent.session_id == original_logger_session_id
|
||||
|
||||
async for _ in agent.act("Do something complex"):
|
||||
pass
|
||||
|
||||
await agent.compact()
|
||||
|
||||
assert agent.session_id != original_session_id
|
||||
assert agent.session_id == agent.session_logger.session_id
|
||||
|
||||
|
||||
class TestAutoCompactIntegration:
|
||||
@pytest.mark.asyncio
|
||||
async def test_auto_compact_triggers_and_preserves_stats(self) -> None:
|
||||
observed: list[tuple[Role, str | None]] = []
|
||||
|
||||
def observer(msg: LLMMessage) -> None:
|
||||
observed.append((msg.role, msg.content))
|
||||
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="<final>")],
|
||||
])
|
||||
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
|
||||
agent = build_test_agent_loop(
|
||||
config=cfg, message_observer=observer, backend=backend
|
||||
)
|
||||
agent.stats.context_tokens = 2
|
||||
|
||||
events = [ev async for ev in agent.act("Hello")]
|
||||
|
||||
assert len(events) == 4
|
||||
assert isinstance(events[0], UserMessageEvent)
|
||||
assert isinstance(events[1], CompactStartEvent)
|
||||
assert isinstance(events[2], CompactEndEvent)
|
||||
assert isinstance(events[3], AssistantEvent)
|
||||
|
||||
start: CompactStartEvent = events[1]
|
||||
final: AssistantEvent = events[3]
|
||||
|
||||
assert start.current_context_tokens == 2
|
||||
assert start.threshold == 1
|
||||
assert final.content == "<final>"
|
||||
|
||||
roles = [r for r, _ in observed]
|
||||
assert roles == [Role.system, Role.user, Role.assistant]
|
||||
assert observed[1][1] == "Hello"
|
||||
|
||||
|
||||
class TestClearHistoryFullReset:
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_preserves_listeners(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
listener_calls: list[int] = []
|
||||
agent.stats.add_listener(
|
||||
"context_tokens", lambda s: listener_calls.append(s.context_tokens)
|
||||
)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
assert agent.stats.context_tokens > 0
|
||||
listener_calls.clear()
|
||||
|
||||
await agent.clear_history()
|
||||
|
||||
assert agent.stats.context_tokens == 0
|
||||
assert any(v == 0 for v in listener_calls)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_fully_resets_stats(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
assert agent.stats.session_prompt_tokens > 0
|
||||
assert agent.stats.steps > 0
|
||||
|
||||
await agent.clear_history()
|
||||
|
||||
assert agent.stats.session_prompt_tokens == 0
|
||||
assert agent.stats.session_completion_tokens == 0
|
||||
assert agent.stats.steps == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_preserves_pricing(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config = make_config(input_price=0.4, output_price=2.0)
|
||||
agent = build_test_agent_loop(config=config, backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
await agent.clear_history()
|
||||
|
||||
assert agent.stats.input_price_per_million == 0.4
|
||||
assert agent.stats.output_price_per_million == 2.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_removes_messages(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
assert len(agent.messages) == 3
|
||||
|
||||
await agent.clear_history()
|
||||
|
||||
assert len(agent.messages) == 1
|
||||
assert agent.messages[0].role == Role.system
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_resets_session_id(self) -> None:
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(disable_logging=False), backend=backend
|
||||
)
|
||||
|
||||
original_session_id = agent.session_id
|
||||
original_logger_session_id = agent.session_logger.session_id
|
||||
|
||||
assert agent.session_id == original_logger_session_id
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
await agent.clear_history()
|
||||
|
||||
assert agent.session_id != original_session_id
|
||||
assert agent.session_id == agent.session_logger.session_id
|
||||
assert agent.parent_session_id is None
|
||||
|
||||
|
||||
class TestClearHistoryObserverBugfix:
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_observer_sees_new_messages(
|
||||
self, observer_capture
|
||||
) -> None:
|
||||
"""Bug fix: clear_history previously left a stale index, so new messages
|
||||
appended after clearing were never observed.
|
||||
"""
|
||||
observed, observer = observer_capture
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="First")],
|
||||
[mock_llm_chunk(content="Second")],
|
||||
])
|
||||
agent = build_test_agent_loop(
|
||||
config=make_config(), message_observer=observer, backend=backend
|
||||
)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
await agent.clear_history()
|
||||
observed.clear()
|
||||
|
||||
async for _ in agent.act("After clear"):
|
||||
pass
|
||||
|
||||
roles = [msg.role for msg in observed]
|
||||
assert Role.user in roles
|
||||
assert Role.assistant in roles
|
||||
|
||||
|
||||
class TestStatsEdgeCases:
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_cost_approximation_on_model_change(
|
||||
self, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
|
||||
|
||||
backend = FakeBackend(mock_llm_chunk(content="Response"))
|
||||
config1 = make_config(active_model="devstral-latest")
|
||||
agent = build_test_agent_loop(config=config1, backend=backend)
|
||||
|
||||
async for _ in agent.act("Hello"):
|
||||
pass
|
||||
|
||||
cost_before = agent.stats.session_cost
|
||||
|
||||
config2 = make_config(active_model="strawberry")
|
||||
await agent.reload_with_initial_messages(base_config=config2)
|
||||
|
||||
cost_after = agent.stats.session_cost
|
||||
|
||||
assert cost_after > cost_before
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_reloads_accumulate_correctly(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="R1")],
|
||||
[mock_llm_chunk(content="R2")],
|
||||
[mock_llm_chunk(content="R3")],
|
||||
])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("First"):
|
||||
pass
|
||||
tokens1 = agent.stats.session_total_llm_tokens
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
async for _ in agent.act("Second"):
|
||||
pass
|
||||
tokens2 = agent.stats.session_total_llm_tokens
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
async for _ in agent.act("Third"):
|
||||
pass
|
||||
tokens3 = agent.stats.session_total_llm_tokens
|
||||
|
||||
assert tokens1 < tokens2 < tokens3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_then_reload_preserves_both(self) -> None:
|
||||
backend = FakeBackend([
|
||||
[mock_llm_chunk(content="Initial response")],
|
||||
[mock_llm_chunk(content="<summary>")],
|
||||
[mock_llm_chunk(content="After reload")],
|
||||
])
|
||||
agent = build_test_agent_loop(config=make_config(), backend=backend)
|
||||
|
||||
async for _ in agent.act("Build something"):
|
||||
pass
|
||||
|
||||
await agent.compact()
|
||||
tokens_after_compact = agent.stats.session_prompt_tokens
|
||||
|
||||
await agent.reload_with_initial_messages()
|
||||
|
||||
assert agent.stats.session_prompt_tokens == tokens_after_compact
|
||||
|
||||
async for _ in agent.act("Continue"):
|
||||
pass
|
||||
|
||||
assert agent.stats.session_prompt_tokens > tokens_after_compact
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_without_config_preserves_current(self) -> None:
|
||||
backend = FakeBackend([])
|
||||
original_config = make_config(active_model="devstral-latest")
|
||||
agent = build_test_agent_loop(config=original_config, backend=backend)
|
||||
|
||||
await agent.reload_with_initial_messages(base_config=None)
|
||||
|
||||
assert agent.config.active_model == "devstral-latest"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_with_new_config_updates_it(self) -> None:
|
||||
backend = FakeBackend([])
|
||||
original_config = make_config(active_model="devstral-latest")
|
||||
agent = build_test_agent_loop(config=original_config, backend=backend)
|
||||
|
||||
new_config = make_config(active_model="devstral-small")
|
||||
await agent.reload_with_initial_messages(base_config=new_config)
|
||||
|
||||
assert agent.config.active_model == "devstral-small"
|
||||
1044
tests/agent_loop/test_agent_tool_call.py
Normal file
1044
tests/agent_loop/test_agent_tool_call.py
Normal file
File diff suppressed because it is too large
Load diff
765
tests/agent_loop/test_agents.py
Normal file
765
tests/agent_loop/test_agents.py
Normal file
|
|
@ -0,0 +1,765 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests import TESTS_ROOT
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from vibe.core.agents.manager import AgentManager
|
||||
from vibe.core.agents.models import (
|
||||
BUILTIN_AGENTS,
|
||||
CHAT,
|
||||
AgentProfile,
|
||||
AgentSafety,
|
||||
AgentType,
|
||||
BuiltinAgentName,
|
||||
_deep_merge,
|
||||
)
|
||||
from vibe.core.config import VibeConfig
|
||||
from vibe.core.prompts import UtilityPrompt
|
||||
from vibe.core.tools.base import ToolPermission
|
||||
from vibe.core.types import LLMChunk, LLMMessage, LLMUsage, Role
|
||||
|
||||
|
||||
class TestDeepMerge:
|
||||
def test_simple_merge(self) -> None:
|
||||
base = {"a": 1, "b": 2}
|
||||
override = {"c": 3}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": 1, "b": 2, "c": 3}
|
||||
|
||||
def test_override_existing_key(self) -> None:
|
||||
base = {"a": 1, "b": 2}
|
||||
override = {"b": 3}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": 1, "b": 3}
|
||||
|
||||
def test_nested_dict_merge(self) -> None:
|
||||
base = {"a": {"x": 1, "y": 2}}
|
||||
override = {"a": {"y": 3, "z": 4}}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": {"x": 1, "y": 3, "z": 4}}
|
||||
|
||||
def test_deeply_nested_merge(self) -> None:
|
||||
base = {"a": {"b": {"c": 1}}}
|
||||
override = {"a": {"b": {"d": 2}}}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": {"b": {"c": 1, "d": 2}}}
|
||||
|
||||
def test_override_dict_with_non_dict(self) -> None:
|
||||
base = {"a": {"x": 1}}
|
||||
override = {"a": "replaced"}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": "replaced"}
|
||||
|
||||
def test_override_non_dict_with_dict(self) -> None:
|
||||
base = {"a": "string"}
|
||||
override = {"a": {"x": 1}}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": {"x": 1}}
|
||||
|
||||
def test_preserves_original_base(self) -> None:
|
||||
base = {"a": 1, "b": {"c": 2}}
|
||||
override = {"b": {"d": 3}}
|
||||
_deep_merge(base, override)
|
||||
assert base == {"a": 1, "b": {"c": 2}}
|
||||
|
||||
def test_empty_override(self) -> None:
|
||||
base = {"a": 1, "b": 2}
|
||||
override: dict = {}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": 1, "b": 2}
|
||||
|
||||
def test_empty_base(self) -> None:
|
||||
base: dict = {}
|
||||
override = {"a": 1}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"a": 1}
|
||||
|
||||
def test_lists_are_overridden_not_merged(self) -> None:
|
||||
"""Lists should be replaced entirely, not merged element-by-element."""
|
||||
base = {"tools": ["read", "grep", "bash"]}
|
||||
override = {"tools": ["write_file"]}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"tools": ["write_file"]}
|
||||
|
||||
def test_nested_lists_are_overridden_not_merged(self) -> None:
|
||||
"""Nested lists in dicts should also be replaced, not merged."""
|
||||
base = {"config": {"enabled_tools": ["a", "b", "c"], "other": 1}}
|
||||
override = {"config": {"enabled_tools": ["x", "y"]}}
|
||||
result = _deep_merge(base, override)
|
||||
assert result == {"config": {"enabled_tools": ["x", "y"], "other": 1}}
|
||||
|
||||
|
||||
class TestAgentSafety:
|
||||
def test_safety_enum_values(self) -> None:
|
||||
assert AgentSafety.SAFE == "safe"
|
||||
assert AgentSafety.NEUTRAL == "neutral"
|
||||
assert AgentSafety.DESTRUCTIVE == "destructive"
|
||||
assert AgentSafety.YOLO == "yolo"
|
||||
|
||||
def test_default_agent_is_neutral(self) -> None:
|
||||
assert BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].safety == AgentSafety.NEUTRAL
|
||||
|
||||
def test_auto_approve_agent_is_yolo(self) -> None:
|
||||
assert BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].safety == AgentSafety.YOLO
|
||||
|
||||
def test_plan_agent_is_safe(self) -> None:
|
||||
assert BUILTIN_AGENTS[BuiltinAgentName.PLAN].safety == AgentSafety.SAFE
|
||||
|
||||
def test_accept_edits_agent_is_destructive(self) -> None:
|
||||
assert (
|
||||
BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].safety
|
||||
== AgentSafety.DESTRUCTIVE
|
||||
)
|
||||
|
||||
|
||||
class TestAgentProfile:
|
||||
def test_all_builtin_agents_have_valid_names(self) -> None:
|
||||
acp_only = {BuiltinAgentName.CHAT}
|
||||
assert set(BUILTIN_AGENTS.keys()) == set(BuiltinAgentName) - acp_only
|
||||
|
||||
def test_display_name_property(self) -> None:
|
||||
assert BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].display_name == "Default"
|
||||
assert (
|
||||
BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].display_name == "Auto Approve"
|
||||
)
|
||||
assert BUILTIN_AGENTS[BuiltinAgentName.PLAN].display_name == "Plan"
|
||||
assert (
|
||||
BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].display_name == "Accept Edits"
|
||||
)
|
||||
|
||||
def test_description_property(self) -> None:
|
||||
assert (
|
||||
"approval" in BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].description.lower()
|
||||
)
|
||||
assert (
|
||||
"auto" in BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].description.lower()
|
||||
)
|
||||
assert "read-only" in BUILTIN_AGENTS[BuiltinAgentName.PLAN].description.lower()
|
||||
assert (
|
||||
"edits" in BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].description.lower()
|
||||
)
|
||||
|
||||
def test_explore_is_subagent(self) -> None:
|
||||
assert BUILTIN_AGENTS[BuiltinAgentName.EXPLORE].agent_type == AgentType.SUBAGENT
|
||||
|
||||
def test_agents(self) -> None:
|
||||
agents = [
|
||||
name
|
||||
for name, profile in BUILTIN_AGENTS.items()
|
||||
if profile.agent_type == AgentType.AGENT
|
||||
]
|
||||
assert set(agents) == {
|
||||
BuiltinAgentName.DEFAULT,
|
||||
BuiltinAgentName.PLAN,
|
||||
BuiltinAgentName.ACCEPT_EDITS,
|
||||
BuiltinAgentName.AUTO_APPROVE,
|
||||
BuiltinAgentName.LEAN,
|
||||
}
|
||||
|
||||
|
||||
class TestAgentApplyToConfig:
|
||||
def test_profile_disabled_tools_are_merged_with_base_config(self) -> None:
|
||||
base = VibeConfig(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_tools=["ask_user_question"],
|
||||
)
|
||||
|
||||
result = BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].apply_to_config(base)
|
||||
|
||||
assert set(result.disabled_tools) == {"ask_user_question", "exit_plan_mode"}
|
||||
|
||||
def test_profile_disabled_tools_preserve_user_disabled_tools(self) -> None:
|
||||
base = VibeConfig(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_tools=["ask_user_question", "custom_tool"],
|
||||
)
|
||||
|
||||
result = BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].apply_to_config(base)
|
||||
|
||||
assert set(result.disabled_tools) == {
|
||||
"ask_user_question",
|
||||
"custom_tool",
|
||||
"exit_plan_mode",
|
||||
}
|
||||
|
||||
def test_base_disabled_tools_are_filtered_from_profile_enabled_tools(self) -> None:
|
||||
base = VibeConfig(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_tools=["ask_user_question"],
|
||||
)
|
||||
|
||||
result = CHAT.apply_to_config(base)
|
||||
|
||||
assert "ask_user_question" not in result.enabled_tools
|
||||
assert "grep" in result.enabled_tools
|
||||
assert "read" in result.enabled_tools
|
||||
assert "task" in result.enabled_tools
|
||||
|
||||
def test_base_disabled_tools_filter_supports_glob_patterns(self) -> None:
|
||||
base = VibeConfig(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_tools=["ask_*"],
|
||||
)
|
||||
agent = AgentProfile(
|
||||
name="custom",
|
||||
display_name="Custom",
|
||||
description="",
|
||||
safety=AgentSafety.NEUTRAL,
|
||||
overrides={"enabled_tools": ["grep", "ask_user_question", "ask_extra"]},
|
||||
)
|
||||
|
||||
result = agent.apply_to_config(base)
|
||||
|
||||
assert result.enabled_tools == ["grep"]
|
||||
|
||||
def test_empty_base_disabled_tools_leaves_enabled_tools_untouched(self) -> None:
|
||||
base = VibeConfig(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_tools=[],
|
||||
)
|
||||
|
||||
result = CHAT.apply_to_config(base)
|
||||
|
||||
assert "ask_user_question" in result.enabled_tools
|
||||
|
||||
def test_custom_prompt_found_in_global_when_missing_from_project(
|
||||
self, mock_prompts_dirs: tuple[Path, Path]
|
||||
) -> None:
|
||||
_, global_prompts = mock_prompts_dirs
|
||||
(global_prompts / "cc.md").write_text("Global custom prompt")
|
||||
|
||||
base = VibeConfig(include_project_context=False, include_prompt_detail=False)
|
||||
agent = AgentProfile(
|
||||
name="cc",
|
||||
display_name="Cc",
|
||||
description="",
|
||||
safety=AgentSafety.NEUTRAL,
|
||||
overrides={"system_prompt_id": "cc"},
|
||||
)
|
||||
result = agent.apply_to_config(base)
|
||||
assert result.system_prompt_id == "cc"
|
||||
assert result.system_prompt == "Global custom prompt"
|
||||
|
||||
def test_custom_prompt_overrides_builtin(
|
||||
self, mock_prompts_dirs: tuple[Path, Path]
|
||||
) -> None:
|
||||
"""Custom prompts in .vibe/prompts/ should override built-in prompts.
|
||||
|
||||
A user-provided explore.md (or any built-in prompt name) in the
|
||||
project or user prompts directory must take priority over the
|
||||
bundled SystemPrompt enum.
|
||||
"""
|
||||
project_prompts, _ = mock_prompts_dirs
|
||||
(project_prompts / "explore.md").write_text("My custom explore prompt")
|
||||
|
||||
config = VibeConfig(
|
||||
system_prompt_id="explore",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
)
|
||||
assert config.system_prompt == "My custom explore prompt"
|
||||
|
||||
def test_custom_compaction_prompt_found_in_global_when_missing_from_project(
|
||||
self, mock_prompts_dirs: tuple[Path, Path]
|
||||
) -> None:
|
||||
_, global_prompts = mock_prompts_dirs
|
||||
(global_prompts / "proofs.md").write_text("Global custom compaction prompt")
|
||||
|
||||
config = VibeConfig(
|
||||
compaction_prompt_id="proofs",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
)
|
||||
assert config.compaction_prompt == "Global custom compaction prompt"
|
||||
|
||||
def test_custom_compaction_prompt_overrides_builtin(
|
||||
self, mock_prompts_dirs: tuple[Path, Path]
|
||||
) -> None:
|
||||
project_prompts, _ = mock_prompts_dirs
|
||||
(project_prompts / "compact.md").write_text("My custom compact prompt")
|
||||
|
||||
config = VibeConfig(
|
||||
compaction_prompt_id="compact",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
)
|
||||
assert config.compaction_prompt == "My custom compact prompt"
|
||||
|
||||
def test_default_compaction_prompt_falls_back_to_builtin(
|
||||
self, mock_prompts_dirs: tuple[Path, Path]
|
||||
) -> None:
|
||||
config = VibeConfig(include_project_context=False, include_prompt_detail=False)
|
||||
assert config.compaction_prompt == UtilityPrompt.COMPACT.read()
|
||||
|
||||
def test_invalid_compaction_prompt_reports_setting_name(
|
||||
self, mock_prompts_dirs: tuple[Path, Path]
|
||||
) -> None:
|
||||
project_prompts, user_prompts = mock_prompts_dirs
|
||||
(project_prompts / "alpha.md").write_text("a")
|
||||
(user_prompts / "beta.md").write_text("b")
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
VibeConfig(
|
||||
compaction_prompt_id="unknown",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
)
|
||||
|
||||
error_text = str(exc_info.value)
|
||||
assert "Invalid compaction_prompt_id value: 'unknown'" in error_text
|
||||
assert 'available prompts ("compact")' in error_text
|
||||
assert '(available: "alpha", "beta")' in error_text
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"malicious_id",
|
||||
["../../../etc/passwd", "..", ".", "subdir/compact", "back\\slash", ""],
|
||||
)
|
||||
def test_prompt_id_rejects_path_traversal(
|
||||
self, mock_prompts_dirs: tuple[Path, Path], malicious_id: str
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="must be a bare filename"):
|
||||
VibeConfig(
|
||||
compaction_prompt_id=malicious_id,
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
)
|
||||
|
||||
|
||||
class TestAgentProfileOverrides:
|
||||
def test_default_agent_disables_exit_plan_mode(self) -> None:
|
||||
overrides = BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].overrides
|
||||
assert "exit_plan_mode" in overrides.get("base_disabled", [])
|
||||
|
||||
def test_auto_approve_agent_sets_bypass_tool_permissions(self) -> None:
|
||||
overrides = BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].overrides
|
||||
assert overrides.get("bypass_tool_permissions") is True
|
||||
|
||||
def test_plan_agent_restricts_tools(self) -> None:
|
||||
overrides = BUILTIN_AGENTS[BuiltinAgentName.PLAN].overrides
|
||||
assert "tools" in overrides
|
||||
tools = overrides["tools"]
|
||||
assert "write_file" in tools
|
||||
assert "edit" in tools
|
||||
assert tools["write_file"]["permission"] == "never"
|
||||
assert tools["edit"]["permission"] == "never"
|
||||
assert len(tools["write_file"]["allowlist"]) > 0
|
||||
assert len(tools["edit"]["allowlist"]) > 0
|
||||
|
||||
def test_accept_edits_agent_sets_tool_permissions(self) -> None:
|
||||
overrides = BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].overrides
|
||||
assert "tools" in overrides
|
||||
tools_config = overrides["tools"]
|
||||
assert "write_file" in tools_config
|
||||
assert "edit" in tools_config
|
||||
assert tools_config["write_file"]["permission"] == "always"
|
||||
assert tools_config["edit"]["permission"] == "always"
|
||||
|
||||
|
||||
class TestAgentManagerCycling:
|
||||
@pytest.fixture
|
||||
def base_config(self) -> VibeConfig:
|
||||
return build_test_vibe_config(
|
||||
include_project_context=False, include_prompt_detail=False
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def backend(self) -> FakeBackend:
|
||||
return FakeBackend([
|
||||
LLMChunk(
|
||||
message=LLMMessage(role=Role.assistant, content="Test response"),
|
||||
usage=LLMUsage(prompt_tokens=10, completion_tokens=5),
|
||||
)
|
||||
])
|
||||
|
||||
def test_get_agent_order_includes_primary_agents(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
|
||||
)
|
||||
order = agent.agent_manager.get_agent_order()
|
||||
assert len(order) == 4
|
||||
assert BuiltinAgentName.DEFAULT in order
|
||||
assert BuiltinAgentName.AUTO_APPROVE in order
|
||||
assert BuiltinAgentName.PLAN in order
|
||||
assert BuiltinAgentName.ACCEPT_EDITS in order
|
||||
|
||||
def test_next_agent_cycles_through_all(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
|
||||
)
|
||||
order = agent.agent_manager.get_agent_order()
|
||||
current = agent.agent_manager.active_profile
|
||||
visited = [current.name]
|
||||
for _ in range(len(order) - 1):
|
||||
current = agent.agent_manager.next_agent(current)
|
||||
visited.append(current.name)
|
||||
assert len(set(visited)) == len(order)
|
||||
|
||||
def test_next_agent_wraps_around(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
|
||||
)
|
||||
order = agent.agent_manager.get_agent_order()
|
||||
last_profile = agent.agent_manager.get_agent(order[-1])
|
||||
first_profile = agent.agent_manager.get_agent(order[0])
|
||||
assert agent.agent_manager.next_agent(last_profile).name == first_profile.name
|
||||
|
||||
|
||||
class TestAgentProfileConfig:
|
||||
def test_agent_profile_frozen(self) -> None:
|
||||
profile = AgentProfile(
|
||||
name="test",
|
||||
display_name="Test",
|
||||
description="Test agent",
|
||||
safety=AgentSafety.NEUTRAL,
|
||||
)
|
||||
with pytest.raises(AttributeError):
|
||||
profile.name = "changed" # pyright: ignore[reportAttributeAccessIssue]
|
||||
|
||||
|
||||
class TestAgentSwitchAgent:
|
||||
@pytest.fixture
|
||||
def base_config(self) -> VibeConfig:
|
||||
return build_test_vibe_config(
|
||||
include_project_context=False, include_prompt_detail=False
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def backend(self) -> FakeBackend:
|
||||
return FakeBackend([
|
||||
LLMChunk(
|
||||
message=LLMMessage(role=Role.assistant, content="Test response"),
|
||||
usage=LLMUsage(prompt_tokens=10, completion_tokens=5),
|
||||
)
|
||||
])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_switch_to_plan_agent_has_tools_with_restricted_permissions(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
|
||||
)
|
||||
await agent.switch_agent(BuiltinAgentName.PLAN)
|
||||
|
||||
plan_tool_names = set(agent.tool_manager.available_tools.keys())
|
||||
# Plan mode now has all tools available but with restricted permissions
|
||||
assert "write_file" in plan_tool_names
|
||||
assert "edit" in plan_tool_names
|
||||
assert "grep" in plan_tool_names
|
||||
assert "read" in plan_tool_names
|
||||
assert agent.agent_profile.name == BuiltinAgentName.PLAN
|
||||
|
||||
# Verify write tools have "never" base permission
|
||||
write_config = agent.tool_manager.get_tool_config("write_file")
|
||||
assert write_config.permission == ToolPermission.NEVER
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_switch_from_plan_to_default_restores_tools(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.PLAN, backend=backend
|
||||
)
|
||||
|
||||
await agent.switch_agent(BuiltinAgentName.DEFAULT)
|
||||
|
||||
# Write tools should revert to default ASK permission
|
||||
write_config = agent.tool_manager.get_tool_config("write_file")
|
||||
assert write_config.permission == ToolPermission.ASK
|
||||
assert agent.agent_profile.name == BuiltinAgentName.DEFAULT
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_switch_agent_preserves_conversation_history(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
|
||||
)
|
||||
user_msg = LLMMessage(role=Role.user, content="Hello")
|
||||
assistant_msg = LLMMessage(role=Role.assistant, content="Hi there")
|
||||
agent.messages.append(user_msg)
|
||||
agent.messages.append(assistant_msg)
|
||||
|
||||
await agent.switch_agent(BuiltinAgentName.PLAN)
|
||||
|
||||
assert len(agent.messages) == 3 # system + user + assistant
|
||||
assert agent.messages[1].content == "Hello"
|
||||
assert agent.messages[2].content == "Hi there"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_switch_to_same_agent_is_noop(
|
||||
self, base_config: VibeConfig, backend: FakeBackend
|
||||
) -> None:
|
||||
agent = build_test_agent_loop(
|
||||
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
|
||||
)
|
||||
original_config = agent.config
|
||||
|
||||
await agent.switch_agent(BuiltinAgentName.DEFAULT)
|
||||
|
||||
assert agent.config is original_config
|
||||
assert agent.agent_profile.name == BuiltinAgentName.DEFAULT
|
||||
|
||||
|
||||
class TestAcceptEditsAgent:
|
||||
def test_accept_edits_config_sets_write_file_always(self) -> None:
|
||||
overrides = BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].overrides
|
||||
assert overrides["tools"]["write_file"]["permission"] == "always"
|
||||
|
||||
def test_accept_edits_config_sets_edit_always(self) -> None:
|
||||
overrides = BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].overrides
|
||||
assert overrides["tools"]["edit"]["permission"] == "always"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accept_edits_agent_auto_approves_write_file(self) -> None:
|
||||
backend = FakeBackend([])
|
||||
|
||||
config = build_test_vibe_config(enabled_tools=["write_file"])
|
||||
agent = build_test_agent_loop(
|
||||
config=config, agent_name=BuiltinAgentName.ACCEPT_EDITS, backend=backend
|
||||
)
|
||||
|
||||
perm = agent.tool_manager.get_tool_config("write_file").permission
|
||||
assert perm == ToolPermission.ALWAYS
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_accept_edits_agent_requires_approval_for_other_tools(self) -> None:
|
||||
backend = FakeBackend([])
|
||||
|
||||
config = build_test_vibe_config(enabled_tools=["bash"])
|
||||
agent = build_test_agent_loop(
|
||||
config=config, agent_name=BuiltinAgentName.ACCEPT_EDITS, backend=backend
|
||||
)
|
||||
|
||||
perm = agent.tool_manager.get_tool_config("bash").permission
|
||||
assert perm == ToolPermission.ASK
|
||||
|
||||
|
||||
class TestPlanAgentToolRestriction:
|
||||
@pytest.mark.asyncio
|
||||
async def test_plan_agent_has_all_tools_with_restricted_write_permissions(
|
||||
self,
|
||||
) -> None:
|
||||
backend = FakeBackend([
|
||||
LLMChunk(
|
||||
message=LLMMessage(role=Role.assistant, content="ok"),
|
||||
usage=LLMUsage(prompt_tokens=10, completion_tokens=5),
|
||||
)
|
||||
])
|
||||
config = build_test_vibe_config()
|
||||
agent = build_test_agent_loop(
|
||||
config=config, agent_name=BuiltinAgentName.PLAN, backend=backend
|
||||
)
|
||||
|
||||
tool_names = set(agent.tool_manager.available_tools.keys())
|
||||
|
||||
# Plan mode now has all tools available
|
||||
assert "grep" in tool_names
|
||||
assert "read" in tool_names
|
||||
assert "write_file" in tool_names
|
||||
assert "edit" in tool_names
|
||||
|
||||
# But write tools have restricted permissions
|
||||
write_config = agent.tool_manager.get_tool_config("write_file")
|
||||
assert write_config.permission == ToolPermission.NEVER
|
||||
assert len(write_config.allowlist) > 0
|
||||
|
||||
edit_config = agent.tool_manager.get_tool_config("edit")
|
||||
assert edit_config.permission == ToolPermission.NEVER
|
||||
assert len(edit_config.allowlist) > 0
|
||||
|
||||
|
||||
class TestAgentManagerFiltering:
|
||||
def test_enabled_agents_filters_to_only_enabled(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
enabled_agents=["default", "plan"],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert len(agents) < len(manager._discovered)
|
||||
assert "default" in agents
|
||||
assert "plan" in agents
|
||||
assert "auto-approve" not in agents
|
||||
assert "accept-edits" not in agents
|
||||
|
||||
def test_disabled_agents_excludes_disabled(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_agents=["auto-approve", "accept-edits"],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert len(agents) < len(manager._discovered)
|
||||
assert "default" in agents
|
||||
assert "plan" in agents
|
||||
assert "auto-approve" not in agents
|
||||
assert "accept-edits" not in agents
|
||||
|
||||
def test_enabled_agents_takes_precedence_over_disabled(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
enabled_agents=["default"],
|
||||
disabled_agents=["default"], # Should be ignored
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert len(agents) == 1
|
||||
assert "default" in agents
|
||||
|
||||
def test_glob_pattern_matching(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_agents=["auto-*", "accept-*"],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert "default" in agents
|
||||
assert "plan" in agents
|
||||
assert "auto-approve" not in agents
|
||||
assert "accept-edits" not in agents
|
||||
|
||||
def test_regex_pattern_matching(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
enabled_agents=["re:^(default|plan)$"],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert len(agents) == 2
|
||||
assert "default" in agents
|
||||
assert "plan" in agents
|
||||
|
||||
def test_empty_enabled_agents_returns_all(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
enabled_agents=[],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert "default" in agents
|
||||
assert "plan" in agents
|
||||
assert "auto-approve" in agents
|
||||
assert "explore" in agents
|
||||
|
||||
def test_install_required_agents_hidden_by_default(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False, include_prompt_detail=False
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert "lean" not in agents
|
||||
|
||||
def test_install_required_agents_visible_when_installed(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
installed_agents=["lean"],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
agents = manager.available_agents
|
||||
assert "lean" in agents
|
||||
|
||||
def test_get_subagents_respects_filtering(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
disabled_agents=["explore"],
|
||||
)
|
||||
manager = AgentManager(lambda: config)
|
||||
|
||||
subagents = manager.get_subagents()
|
||||
names = [a.name for a in subagents]
|
||||
assert "explore" not in names
|
||||
|
||||
|
||||
class TestAgentLoopInitialization:
|
||||
def test_agent_system_prompt_id_is_applied_on_init(
|
||||
self, mock_prompts_dirs: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_, global_prompts = mock_prompts_dirs
|
||||
custom_prompt_content = "CUSTOM_AGENT_PROMPT_MARKER"
|
||||
(global_prompts / "custom_agent.md").write_text(custom_prompt_content)
|
||||
|
||||
custom_agent = AgentProfile(
|
||||
name="custom_test_agent",
|
||||
display_name="Custom Test",
|
||||
description="Test agent with custom system prompt",
|
||||
safety=AgentSafety.NEUTRAL,
|
||||
overrides={"system_prompt_id": "custom_agent"},
|
||||
)
|
||||
patched_agents = {**BUILTIN_AGENTS, "custom_test_agent": custom_agent}
|
||||
monkeypatch.setattr("vibe.core.agents.models.BUILTIN_AGENTS", patched_agents)
|
||||
monkeypatch.setattr("vibe.core.agents.manager.BUILTIN_AGENTS", patched_agents)
|
||||
|
||||
config = build_test_vibe_config(
|
||||
include_project_context=False, include_prompt_detail=False
|
||||
)
|
||||
assert config.system_prompt_id == "cli", (
|
||||
"Base config should use default 'cli' prompt"
|
||||
)
|
||||
|
||||
agent_loop = build_test_agent_loop(
|
||||
config=config, agent_name="custom_test_agent"
|
||||
)
|
||||
|
||||
assert agent_loop.config.system_prompt_id == "custom_agent", (
|
||||
"Merged config should have the agent's system_prompt_id override"
|
||||
)
|
||||
|
||||
system_message = agent_loop.messages[0]
|
||||
assert system_message.role == Role.system
|
||||
assert system_message.content is not None
|
||||
assert custom_prompt_content in system_message.content, (
|
||||
f"System message should contain custom prompt content. "
|
||||
f"Expected '{custom_prompt_content}' to be in system message."
|
||||
)
|
||||
|
||||
|
||||
class TestActConsumersUseAclosing:
|
||||
def test_no_bare_async_for_over_act(self) -> None:
|
||||
vibe_pkg = TESTS_ROOT.parent / "vibe"
|
||||
violations: list[str] = []
|
||||
for path in vibe_pkg.rglob("*.py"):
|
||||
tree = ast.parse(path.read_text(), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.AsyncFor):
|
||||
continue
|
||||
match node.iter:
|
||||
case ast.Call(func=ast.Attribute(attr="act")):
|
||||
violations.append(f"{path}:{node.lineno}")
|
||||
|
||||
assert not violations, (
|
||||
"Bare `async for ... in .act()` found — wrap in "
|
||||
"contextlib.aclosing(). See issue #569.\n" + "\n".join(violations)
|
||||
)
|
||||
122
tests/agent_loop/test_approve_always_permanent.py
Normal file
122
tests/agent_loop/test_approve_always_permanent.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import tomllib
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from vibe.core.tools.base import ToolPermission
|
||||
from vibe.core.tools.permissions import PermissionScope, RequiredPermission
|
||||
|
||||
|
||||
def _read_persisted_config(config_dir: Path) -> dict:
|
||||
config_file = config_dir / "config.toml"
|
||||
with config_file.open("rb") as f:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
class TestApproveAlwaysPermanentNoGranularPermissions:
|
||||
def test_sets_tool_permission_always_in_config(self, config_dir: Path):
|
||||
agent = build_test_agent_loop()
|
||||
|
||||
agent.approve_always("bash", None, save_permanently=True)
|
||||
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
assert persisted["tools"]["bash"]["permission"] == "always"
|
||||
|
||||
def test_session_only_does_not_persist(self, config_dir: Path):
|
||||
agent = build_test_agent_loop()
|
||||
|
||||
agent.approve_always("bash", None, save_permanently=False)
|
||||
|
||||
assert (
|
||||
agent.tool_manager.get_tool_config("bash").permission
|
||||
== ToolPermission.ALWAYS
|
||||
)
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
assert "bash" not in persisted.get("tools", {})
|
||||
|
||||
|
||||
class TestApproveAlwaysPermanentWithGranularPermissions:
|
||||
def _make_permissions(self) -> list[RequiredPermission]:
|
||||
return [
|
||||
RequiredPermission(
|
||||
scope=PermissionScope.COMMAND_PATTERN,
|
||||
invocation_pattern="npm install foo",
|
||||
session_pattern="npm install *",
|
||||
label="npm install *",
|
||||
)
|
||||
]
|
||||
|
||||
def test_persists_allowlist_to_config(self, config_dir: Path):
|
||||
agent = build_test_agent_loop()
|
||||
perms = self._make_permissions()
|
||||
|
||||
agent.approve_always("bash", perms, save_permanently=True)
|
||||
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
assert persisted["tools"]["bash"]["allowlist"] == ["npm install"]
|
||||
|
||||
def test_also_adds_session_rules(self, config_dir: Path):
|
||||
agent = build_test_agent_loop()
|
||||
perms = self._make_permissions()
|
||||
|
||||
agent.approve_always("bash", perms, save_permanently=True)
|
||||
|
||||
assert len(agent._permission_store._rules) == 1
|
||||
rule = agent._permission_store._rules[0]
|
||||
assert rule.tool_name == "bash"
|
||||
assert rule.scope == PermissionScope.COMMAND_PATTERN
|
||||
assert rule.session_pattern == "npm install *"
|
||||
|
||||
def test_session_only_does_not_persist_allowlist(self, config_dir: Path):
|
||||
agent = build_test_agent_loop()
|
||||
perms = self._make_permissions()
|
||||
|
||||
agent.approve_always("bash", perms, save_permanently=False)
|
||||
|
||||
assert len(agent._permission_store._rules) == 1
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
assert "bash" not in persisted.get("tools", {})
|
||||
|
||||
def test_does_not_duplicate_existing_allowlist_entries(self, config_dir: Path):
|
||||
config = build_test_vibe_config(tools={"bash": {"allowlist": ["npm install"]}})
|
||||
agent = build_test_agent_loop(config=config)
|
||||
perms = self._make_permissions()
|
||||
|
||||
agent.approve_always("bash", perms, save_permanently=True)
|
||||
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
# Pattern already existed -- nothing new should be written
|
||||
assert persisted.get("tools", {}).get("bash", {}).get("allowlist") is None
|
||||
|
||||
def test_appends_new_patterns_to_existing_allowlist(self, config_dir: Path):
|
||||
config = build_test_vibe_config(tools={"bash": {"allowlist": ["git"]}})
|
||||
agent = build_test_agent_loop(config=config)
|
||||
perms = self._make_permissions()
|
||||
|
||||
agent.approve_always("bash", perms, save_permanently=True)
|
||||
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
assert persisted["tools"]["bash"]["allowlist"] == ["git", "npm install"]
|
||||
|
||||
def test_multiple_permissions_persisted(self, config_dir: Path):
|
||||
agent = build_test_agent_loop()
|
||||
perms = [
|
||||
RequiredPermission(
|
||||
scope=PermissionScope.COMMAND_PATTERN,
|
||||
invocation_pattern="npm install foo",
|
||||
session_pattern="npm install *",
|
||||
label="npm install *",
|
||||
),
|
||||
RequiredPermission(
|
||||
scope=PermissionScope.OUTSIDE_DIRECTORY,
|
||||
invocation_pattern="/tmp/newdir",
|
||||
session_pattern="/tmp/*",
|
||||
label="/tmp/*",
|
||||
),
|
||||
]
|
||||
|
||||
agent.approve_always("bash", perms, save_permanently=True)
|
||||
|
||||
persisted = _read_persisted_config(config_dir)
|
||||
assert persisted["tools"]["bash"]["allowlist"] == ["/tmp/*", "npm install"]
|
||||
593
tests/agent_loop/test_deferred_init.py
Normal file
593
tests/agent_loop/test_deferred_init.py
Normal file
|
|
@ -0,0 +1,593 @@
|
|||
"""Tests for deferred initialization: _complete_init, _wait_for_init, integrate_mcp idempotency."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.mock.utils import mock_llm_chunk
|
||||
from tests.stubs.fake_backend import FakeBackend
|
||||
from tests.stubs.fake_connector_registry import FakeConnectorRegistry
|
||||
from tests.stubs.fake_mcp_registry import FakeMCPRegistry
|
||||
from vibe.core import agent_loop as agent_loop_module
|
||||
from vibe.core.agent_loop import AgentLoop
|
||||
from vibe.core.config import MCPStdio
|
||||
from vibe.core.telemetry.types import TerminalEmulator
|
||||
from vibe.core.tools.manager import ToolManager
|
||||
from vibe.core.tools.mcp import AuthStatus
|
||||
from vibe.core.tools.mcp.tools import RemoteTool
|
||||
|
||||
|
||||
def _build_uninitiated_loop(**kwargs):
|
||||
"""Build a test loop with defer_heavy_init=True but without auto-starting the init thread."""
|
||||
with patch.object(AgentLoop, "_start_deferred_init"):
|
||||
return build_test_agent_loop(defer_heavy_init=True, **kwargs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _complete_init
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_init(loop: AgentLoop) -> None:
|
||||
"""Run _complete_init in a thread (matching production behavior) and wait."""
|
||||
thread = threading.Thread(target=loop._complete_init, daemon=True)
|
||||
loop._deferred_init_thread = thread
|
||||
thread.start()
|
||||
thread.join()
|
||||
|
||||
|
||||
class TestCompleteInit:
|
||||
def test_success_sets_init_complete(self) -> None:
|
||||
loop = _build_uninitiated_loop()
|
||||
assert not loop.is_initialized
|
||||
|
||||
_run_init(loop)
|
||||
|
||||
assert loop.is_initialized
|
||||
assert loop._init_error is None
|
||||
|
||||
def test_failure_sets_init_complete_and_stores_error(self) -> None:
|
||||
loop = _build_uninitiated_loop()
|
||||
error = RuntimeError("mcp boom")
|
||||
|
||||
with patch.object(loop.tool_manager, "integrate_all", side_effect=error):
|
||||
_run_init(loop)
|
||||
|
||||
assert loop.is_initialized
|
||||
assert loop._init_error is error
|
||||
|
||||
def test_mcp_failure_sets_init_error(self) -> None:
|
||||
mcp_server = MCPStdio(name="test-server", transport="stdio", command="echo")
|
||||
config = build_test_vibe_config(mcp_servers=[mcp_server])
|
||||
loop = _build_uninitiated_loop(config=config)
|
||||
|
||||
with patch.object(
|
||||
loop.tool_manager._mcp_registry,
|
||||
"get_tools_async",
|
||||
side_effect=RuntimeError("mcp discovery boom"),
|
||||
):
|
||||
_run_init(loop)
|
||||
|
||||
assert loop.is_initialized
|
||||
assert isinstance(loop._init_error, RuntimeError)
|
||||
assert str(loop._init_error) == "mcp discovery boom"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# wait_until_ready
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestWaitForInit:
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_immediately_when_already_complete(self) -> None:
|
||||
loop = build_test_agent_loop(defer_heavy_init=True)
|
||||
|
||||
await loop.wait_until_ready() # should not block
|
||||
|
||||
assert loop.is_initialized
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_waits_for_background_thread(self) -> None:
|
||||
loop = build_test_agent_loop(defer_heavy_init=True)
|
||||
|
||||
await loop.wait_until_ready()
|
||||
|
||||
assert loop.is_initialized
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_stored_error(self) -> None:
|
||||
loop = _build_uninitiated_loop()
|
||||
error = RuntimeError("init failed")
|
||||
|
||||
with patch.object(loop.tool_manager, "integrate_all", side_effect=error):
|
||||
loop._complete_init()
|
||||
|
||||
with pytest.raises(RuntimeError, match="init failed"):
|
||||
await loop.wait_until_ready()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raises_error_for_every_caller(self) -> None:
|
||||
loop = _build_uninitiated_loop()
|
||||
error = RuntimeError("once only")
|
||||
|
||||
with patch.object(loop.tool_manager, "integrate_all", side_effect=error):
|
||||
loop._complete_init()
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
await loop.wait_until_ready()
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
await loop.wait_until_ready()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# integrate_mcp idempotency
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIntegrateMcpIdempotency:
|
||||
def test_second_call_is_noop(self) -> None:
|
||||
mcp_server = MCPStdio(name="test-server", transport="stdio", command="echo")
|
||||
config = build_test_vibe_config(mcp_servers=[mcp_server])
|
||||
registry = FakeMCPRegistry()
|
||||
manager = ToolManager(lambda: config, mcp_registry=registry, defer_mcp=True)
|
||||
|
||||
manager.integrate_mcp()
|
||||
tools_after_first = dict(manager.registered_tools)
|
||||
|
||||
# Spy on the registry to ensure get_tools is not called again.
|
||||
registry.get_tools = MagicMock(wraps=registry.get_tools)
|
||||
manager.integrate_mcp()
|
||||
|
||||
registry.get_tools.assert_not_called()
|
||||
assert manager.registered_tools == tools_after_first
|
||||
|
||||
def test_flag_not_set_when_no_servers(self) -> None:
|
||||
config = build_test_vibe_config(mcp_servers=[])
|
||||
manager = ToolManager(lambda: config, defer_mcp=True)
|
||||
|
||||
manager.integrate_mcp()
|
||||
|
||||
# No servers means the method returns early without setting the flag,
|
||||
# so a future call with servers would still run discovery.
|
||||
assert not manager._mcp_integrated
|
||||
|
||||
def test_no_servers_syncs_shared_registry_status(self) -> None:
|
||||
config = build_test_vibe_config(
|
||||
mcp_servers=[MCPStdio(name="srv", transport="stdio", command="echo")]
|
||||
)
|
||||
registry = FakeMCPRegistry()
|
||||
manager = ToolManager(lambda: config, mcp_registry=registry, defer_mcp=True)
|
||||
|
||||
manager.integrate_mcp()
|
||||
assert registry.status() == {"srv": AuthStatus.STDIO}
|
||||
|
||||
config = build_test_vibe_config(mcp_servers=[])
|
||||
manager = ToolManager(lambda: config, mcp_registry=registry, defer_mcp=True)
|
||||
manager.integrate_mcp()
|
||||
|
||||
assert registry.status() == {}
|
||||
assert not manager._mcp_integrated
|
||||
|
||||
|
||||
class TestRefreshRemoteTools:
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_rediscovers_mcp_and_connector_tools(self) -> None:
|
||||
mcp_server = MCPStdio(name="srv", transport="stdio", command="echo")
|
||||
config = build_test_vibe_config(mcp_servers=[mcp_server])
|
||||
registry = FakeMCPRegistry()
|
||||
registry.get_tools_async = AsyncMock(wraps=registry.get_tools_async)
|
||||
connector_registry = FakeConnectorRegistry({
|
||||
"alpha": [RemoteTool(name="search", description="Search alpha")]
|
||||
})
|
||||
manager = ToolManager(
|
||||
lambda: config,
|
||||
mcp_registry=registry,
|
||||
connector_registry=connector_registry,
|
||||
defer_mcp=True,
|
||||
)
|
||||
|
||||
await manager.refresh_remote_tools_async()
|
||||
|
||||
assert "srv_fake_tool" in manager.registered_tools
|
||||
assert "connector_alpha_search" in manager.registered_tools
|
||||
|
||||
connector_registry._fake_connectors = {
|
||||
"beta": [RemoteTool(name="list", description="List beta")]
|
||||
}
|
||||
|
||||
await manager.refresh_remote_tools_async()
|
||||
|
||||
assert registry.get_tools_async.await_count == 2
|
||||
assert "srv_fake_tool" in manager.registered_tools
|
||||
assert "connector_alpha_search" not in manager.registered_tools
|
||||
assert "connector_beta_list" in manager.registered_tools
|
||||
|
||||
|
||||
class TestDeferredInitPublicMethods:
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_waits_for_deferred_init(self) -> None:
|
||||
loop = build_test_agent_loop(
|
||||
defer_heavy_init=True, backend=FakeBackend(mock_llm_chunk(content="hello"))
|
||||
)
|
||||
|
||||
events = [event async for event in loop.act("Hello")]
|
||||
|
||||
assert loop.is_initialized
|
||||
assert [event.content for event in events if hasattr(event, "content")][
|
||||
-1
|
||||
] == "hello"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reload_with_initial_messages_waits_for_deferred_init(self) -> None:
|
||||
loop = build_test_agent_loop(defer_heavy_init=True)
|
||||
|
||||
await loop.reload_with_initial_messages()
|
||||
|
||||
assert loop.is_initialized
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_switch_agent_waits_for_deferred_init(self) -> None:
|
||||
loop = build_test_agent_loop(defer_heavy_init=True)
|
||||
|
||||
await loop.switch_agent("plan")
|
||||
|
||||
assert loop.is_initialized
|
||||
assert loop.agent_profile.name == "plan"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_history_waits_for_deferred_init(self) -> None:
|
||||
loop = build_test_agent_loop(
|
||||
defer_heavy_init=True, backend=FakeBackend(mock_llm_chunk(content="hello"))
|
||||
)
|
||||
[_ async for _ in loop.act("Hello")]
|
||||
|
||||
await loop.clear_history()
|
||||
|
||||
assert loop.is_initialized
|
||||
assert len(loop.messages) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compact_waits_for_deferred_init(self) -> None:
|
||||
loop = build_test_agent_loop(
|
||||
defer_heavy_init=True,
|
||||
backend=FakeBackend([
|
||||
[mock_llm_chunk(content="hello")],
|
||||
[mock_llm_chunk(content="summary")],
|
||||
]),
|
||||
)
|
||||
[_ async for _ in loop.act("Hello")]
|
||||
|
||||
summary = await loop.compact()
|
||||
|
||||
assert loop.is_initialized
|
||||
assert summary == "summary"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_inject_user_context_waits_for_deferred_init(self) -> None:
|
||||
loop = build_test_agent_loop(defer_heavy_init=True)
|
||||
|
||||
await loop.inject_user_context("context")
|
||||
|
||||
assert loop.is_initialized
|
||||
assert loop.messages[-1].content == "context"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# start_initialize_experiments / wait_until_ready experiment gating
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStartInitializeExperiments:
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_block_caller(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def slow_init() -> None:
|
||||
await gate.wait()
|
||||
|
||||
with patch.object(loop, "initialize_experiments", side_effect=slow_init):
|
||||
loop.start_initialize_experiments()
|
||||
|
||||
task = loop._experiments_task
|
||||
assert task is not None
|
||||
assert not task.done()
|
||||
|
||||
gate.set()
|
||||
await task
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_idempotent(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
init_mock = AsyncMock()
|
||||
|
||||
with patch.object(loop, "initialize_experiments", new=init_mock):
|
||||
loop.start_initialize_experiments()
|
||||
first_task = loop._experiments_task
|
||||
loop.start_initialize_experiments()
|
||||
second_task = loop._experiments_task
|
||||
|
||||
assert first_task is second_task
|
||||
assert first_task is not None
|
||||
await first_task
|
||||
|
||||
assert init_mock.await_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sets_pending_telemetry_flags(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
|
||||
with patch.object(loop, "initialize_experiments", new=AsyncMock()):
|
||||
loop.start_initialize_experiments()
|
||||
|
||||
assert loop._pending_new_session_telemetry is True
|
||||
assert loop._ready_telemetry_pending is True
|
||||
|
||||
task = loop._experiments_task
|
||||
assert task is not None
|
||||
await task
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refreshes_system_prompt_when_experiments_update(self) -> None:
|
||||
loop = build_test_agent_loop(terminal_emulator=TerminalEmulator.VSCODE)
|
||||
refresh_mock = AsyncMock()
|
||||
init_mock = AsyncMock(return_value=True)
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
agent_loop_module, "session_initialize_experiments", new=init_mock
|
||||
),
|
||||
patch.object(loop, "refresh_system_prompt", new=refresh_mock),
|
||||
):
|
||||
await loop.initialize_experiments()
|
||||
|
||||
refresh_mock.assert_awaited_once()
|
||||
init_mock.assert_awaited_once()
|
||||
init_args = init_mock.await_args
|
||||
assert init_args is not None
|
||||
assert init_args.kwargs["terminal_emulator"] is TerminalEmulator.VSCODE
|
||||
|
||||
def test_new_session_telemetry_uses_provided_terminal_emulator(self) -> None:
|
||||
loop = build_test_agent_loop(terminal_emulator=TerminalEmulator.VSCODE)
|
||||
send_new_session = MagicMock()
|
||||
|
||||
with patch.object(
|
||||
loop.telemetry_client, "send_new_session", new=send_new_session
|
||||
):
|
||||
loop.emit_new_session_telemetry()
|
||||
|
||||
assert (
|
||||
send_new_session.call_args.kwargs["terminal_emulator"]
|
||||
is TerminalEmulator.VSCODE
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_refresh_system_prompt_when_experiments_unchanged(
|
||||
self,
|
||||
) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
refresh_mock = AsyncMock()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
agent_loop_module,
|
||||
"session_initialize_experiments",
|
||||
new=AsyncMock(return_value=False),
|
||||
),
|
||||
patch.object(loop, "refresh_system_prompt", new=refresh_mock),
|
||||
):
|
||||
await loop.initialize_experiments()
|
||||
|
||||
refresh_mock.assert_not_awaited()
|
||||
|
||||
|
||||
class TestWaitUntilReadyJoinsExperiments:
|
||||
@pytest.mark.asyncio
|
||||
async def test_joins_in_flight_task(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
gate = asyncio.Event()
|
||||
completed = False
|
||||
|
||||
async def slow_init() -> None:
|
||||
nonlocal completed
|
||||
await gate.wait()
|
||||
completed = True
|
||||
|
||||
with patch.object(loop, "initialize_experiments", side_effect=slow_init):
|
||||
loop.start_initialize_experiments()
|
||||
|
||||
async def release() -> None:
|
||||
await asyncio.sleep(0.01)
|
||||
gate.set()
|
||||
|
||||
asyncio.create_task(release())
|
||||
await loop.wait_until_ready()
|
||||
|
||||
assert completed is True
|
||||
task = loop._experiments_task
|
||||
assert task is not None
|
||||
assert task.done()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_emits_new_session_telemetry_once(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
emit_new_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(loop, "initialize_experiments", new=AsyncMock()),
|
||||
patch.object(loop, "emit_new_session_telemetry", new=emit_new_session),
|
||||
):
|
||||
loop.start_initialize_experiments()
|
||||
await loop.wait_until_ready()
|
||||
await loop.wait_until_ready()
|
||||
|
||||
emit_new_session.assert_called_once()
|
||||
assert loop._pending_new_session_telemetry is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_emits_ready_telemetry_when_only_experiments_deferred(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
emit_ready = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(loop, "initialize_experiments", new=AsyncMock()),
|
||||
patch.object(loop, "emit_ready_telemetry", new=emit_ready),
|
||||
):
|
||||
loop.start_initialize_experiments()
|
||||
await loop.wait_until_ready()
|
||||
await loop.wait_until_ready()
|
||||
|
||||
emit_ready.assert_called_once()
|
||||
((duration,), _) = emit_ready.call_args
|
||||
assert isinstance(duration, int)
|
||||
assert duration >= 0
|
||||
assert loop._ready_telemetry_pending is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_emit_new_session_when_only_hydrating(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
emit_new_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(loop, "hydrate_experiments_from_session", new=AsyncMock()),
|
||||
patch.object(loop, "emit_new_session_telemetry", new=emit_new_session),
|
||||
):
|
||||
await loop.hydrate_experiments_from_session()
|
||||
await loop.wait_until_ready()
|
||||
|
||||
emit_new_session.assert_not_called()
|
||||
assert loop._pending_new_session_telemetry is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_op_when_nothing_deferred(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
emit_ready = MagicMock()
|
||||
emit_new_session = MagicMock()
|
||||
|
||||
with (
|
||||
patch.object(loop, "emit_ready_telemetry", new=emit_ready),
|
||||
patch.object(loop, "emit_new_session_telemetry", new=emit_new_session),
|
||||
):
|
||||
await loop.wait_until_ready()
|
||||
|
||||
emit_ready.assert_not_called()
|
||||
emit_new_session.assert_not_called()
|
||||
|
||||
|
||||
class TestACloseCancelsExperimentsTask:
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancels_in_flight_task(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def never_completing() -> None:
|
||||
await gate.wait()
|
||||
|
||||
with patch.object(loop, "initialize_experiments", side_effect=never_completing):
|
||||
loop.start_initialize_experiments()
|
||||
task = loop._experiments_task
|
||||
assert task is not None
|
||||
assert not task.done()
|
||||
|
||||
await loop.aclose()
|
||||
|
||||
assert task.done()
|
||||
assert task.cancelled()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_cancel_completed_task(self) -> None:
|
||||
loop = build_test_agent_loop()
|
||||
|
||||
with patch.object(loop, "initialize_experiments", new=AsyncMock()):
|
||||
loop.start_initialize_experiments()
|
||||
task = loop._experiments_task
|
||||
assert task is not None
|
||||
await task
|
||||
|
||||
await loop.aclose()
|
||||
|
||||
assert task.done()
|
||||
assert not task.cancelled()
|
||||
|
||||
|
||||
class TestCycleAgentDuringInit:
|
||||
@pytest.mark.asyncio
|
||||
async def test_shift_tab_during_experiments_init_does_not_crash(self) -> None:
|
||||
"""Regression: shift+tab during init crashed with
|
||||
RuntimeError("await wasn't used with future").
|
||||
|
||||
_cycle_agent ran switch_agent via asyncio.run() in a thread worker,
|
||||
creating a second event loop. wait_until_ready then tried to await
|
||||
_experiments_task (owned by the main Textual loop) from that new loop.
|
||||
|
||||
The fix uses asyncio.run_coroutine_threadsafe() to schedule on the
|
||||
main loop instead. This test presses shift+tab while experiments
|
||||
are still initializing and asserts the worker completes without error.
|
||||
"""
|
||||
from tests.conftest import build_test_vibe_app
|
||||
|
||||
gate = asyncio.Event()
|
||||
|
||||
async def slow_init() -> None:
|
||||
await gate.wait()
|
||||
|
||||
agent_loop = build_test_agent_loop()
|
||||
app = build_test_vibe_app(agent_loop=agent_loop)
|
||||
|
||||
async with app.run_test() as pilot:
|
||||
with patch.object(
|
||||
agent_loop, "initialize_experiments", side_effect=slow_init
|
||||
):
|
||||
agent_loop._experiments_task = None # reset so start_ re-fires
|
||||
agent_loop.start_initialize_experiments()
|
||||
|
||||
assert agent_loop._experiments_task is not None
|
||||
assert not agent_loop._experiments_task.done()
|
||||
|
||||
# Press shift+tab while experiments are still running.
|
||||
await pilot.press("shift+tab")
|
||||
await pilot.pause(0.05)
|
||||
|
||||
# Unblock experiments so switch_agent can complete.
|
||||
gate.set()
|
||||
|
||||
# wait_for_complete raises WorkerFailed if the thread worker
|
||||
# crashed — which is exactly what happened before the fix.
|
||||
await pilot.app.workers.wait_for_complete()
|
||||
|
||||
assert agent_loop.agent_profile.name == "plan"
|
||||
|
||||
|
||||
class TestActGatesOnExperiments:
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_awaits_experiments_before_llm_call(self) -> None:
|
||||
loop = build_test_agent_loop(
|
||||
backend=FakeBackend(mock_llm_chunk(content="hello"))
|
||||
)
|
||||
gate = asyncio.Event()
|
||||
finished_init = False
|
||||
|
||||
async def slow_init() -> None:
|
||||
nonlocal finished_init
|
||||
await gate.wait()
|
||||
finished_init = True
|
||||
|
||||
with patch.object(loop, "initialize_experiments", side_effect=slow_init):
|
||||
loop.start_initialize_experiments()
|
||||
|
||||
async def release() -> None:
|
||||
await asyncio.sleep(0.01)
|
||||
gate.set()
|
||||
|
||||
asyncio.create_task(release())
|
||||
events = [event async for event in loop.act("Hello")]
|
||||
|
||||
assert finished_init is True
|
||||
assert any(getattr(event, "content", None) == "hello" for event in events)
|
||||
Loading…
Add table
Add a link
Reference in a new issue