Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Hdandria <henri.dandria@mistral.ai>
Co-authored-by: Ivana Dunisijevic <ivana.dunisijevic@mistral.ai>
Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Mert Unsal <mert.unsal@mistral.ai>
Co-authored-by: Michel Thomazo <51709227+michelTho@users.noreply.github.com>
Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Val <102326092+vdeva@users.noreply.github.com>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: renovate-mistral[bot] <253709520+renovate-mistral[bot]@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Clément Drouin 2026-06-19 11:01:24 +02:00 committed by GitHub
parent 564a14365e
commit 6bedf271ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
223 changed files with 10533 additions and 6947 deletions

View file

View file

@ -0,0 +1,181 @@
from __future__ import annotations
from collections.abc import Iterator
import json
from typing import Any
import httpx
import pytest
import respx
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.constants import (
ANTHROPIC_BASE_URL,
ANTHROPIC_MESSAGES_PATH,
CHAT_COMPLETIONS_PATH,
CONNECTORS_BOOTSTRAP_PATH,
MISTRAL_BASE_URL,
OPENAI_BASE_URL,
OPENAI_RESPONSES_PATH,
)
from vibe.core.agent_loop import AgentLoop
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.llm.backend.factory import BACKEND_FACTORY
from vibe.core.types import AssistantEvent, Backend, BaseEvent
def assistant_text(events: list[BaseEvent]) -> str:
return "".join(
e.content for e in events if isinstance(e, AssistantEvent) and e.content
).strip()
@pytest.fixture(autouse=True)
def _git_offline(monkeypatch: pytest.MonkeyPatch) -> None:
# Restrict git to the local file protocol so no test can fetch/push over the
# network, even if a remote is misconfigured to a real URL.
monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file")
def e2e_config(**overrides: Any) -> VibeConfig:
# Defaults give the mistral provider/model (api.mistral.ai, the respx-mocked
# base). The default model reasons, but that only adds reasoning_effort to the
# request, which the mocked responses ignore.
# The e2e harness mocks the connector bootstrap endpoint via respx, so
# connector discovery is enabled (and fast) here unlike the unit-test default.
return build_test_vibe_config(
enabled_tools=overrides.pop("enabled_tools", []),
enable_connectors=overrides.pop("enable_connectors", True),
system_prompt_id="tests",
include_project_context=False,
include_prompt_detail=False,
**overrides,
)
def _generic_e2e_config(
*, name: str, api_base: str, api_style: str, model: str, **overrides: Any
) -> VibeConfig:
provider = ProviderConfig(
name=name,
api_base=api_base,
api_key_env_var=f"{name.upper()}_API_KEY",
api_style=api_style,
backend=Backend.GENERIC,
)
models = [ModelConfig(name=model, provider=name, alias=name)]
return e2e_config(
active_model=name, models=models, providers=[provider], **overrides
)
def anthropic_e2e_config(**overrides: Any) -> VibeConfig:
return _generic_e2e_config(
name="anthropic",
api_base=ANTHROPIC_BASE_URL,
api_style="anthropic",
model="claude-test",
**overrides,
)
def openai_responses_e2e_config(**overrides: Any) -> VibeConfig:
return _generic_e2e_config(
name="openai",
api_base=f"{OPENAI_BASE_URL}/v1",
api_style="openai-responses",
model="gpt-test",
**overrides,
)
@pytest.fixture
def mock_mistral() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=MISTRAL_BASE_URL, assert_all_called=False) as router:
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
return_value=httpx.Response(200, json={"connectors": []})
)
yield router
def build_e2e_agent_loop(
*, config: VibeConfig | None = None, enable_streaming: bool = False, **kwargs: Any
) -> AgentLoop:
resolved_config = config or e2e_config()
provider = resolved_config.providers[0]
# todo now we use build_test_agent_loop for the fake mcp registry.
# Maybe mock http tool calls and instantiate a real registry later for more coverage
return build_test_agent_loop(
config=resolved_config,
agent_name=kwargs.pop("agent_name", BuiltinAgentName.AUTO_APPROVE),
backend=BACKEND_FACTORY[provider.backend](provider=provider),
enable_streaming=enable_streaming,
**kwargs,
)
class ProviderAPI:
"""Stubs a provider's completion wire; the AgentLoop stays the subject."""
def __init__(self, router: respx.MockRouter, path: str) -> None:
self.route = router.post(path)
def reply(self, *completions: dict[str, Any]) -> None:
responses = [httpx.Response(200, json=c) for c in completions]
if len(responses) == 1:
self.route.mock(return_value=responses[0])
else:
self.route.mock(side_effect=responses)
def reply_stream(self, chunks: list[bytes]) -> None:
self.route.mock(return_value=self._stream_response(chunks))
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
self.route.mock(
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
)
@staticmethod
def _stream_response(chunks: list[bytes]) -> httpx.Response:
return httpx.Response(
200,
stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
headers={"Content-Type": "text/event-stream"},
)
@property
def request_json(self) -> dict[str, Any]:
return json.loads(self.route.calls.last.request.content)
class MistralAPI(ProviderAPI):
def __init__(self, router: respx.MockRouter) -> None:
super().__init__(router, CHAT_COMPLETIONS_PATH)
@pytest.fixture
def mistral_api(mock_mistral: respx.MockRouter) -> MistralAPI:
return MistralAPI(mock_mistral)
@pytest.fixture
def mock_anthropic() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=ANTHROPIC_BASE_URL, assert_all_called=False) as router:
yield router
@pytest.fixture
def anthropic_api(mock_anthropic: respx.MockRouter) -> ProviderAPI:
return ProviderAPI(mock_anthropic, ANTHROPIC_MESSAGES_PATH)
@pytest.fixture
def mock_openai() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=OPENAI_BASE_URL, assert_all_called=False) as router:
yield router
@pytest.fixture
def openai_responses_api(mock_openai: respx.MockRouter) -> ProviderAPI:
return ProviderAPI(mock_openai, OPENAI_RESPONSES_PATH)

View file

@ -0,0 +1,111 @@
from __future__ import annotations
from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import (
MistralAPI,
assistant_text,
build_e2e_agent_loop,
e2e_config,
)
from tests.backend.data.mistral import (
STREAMED_SIMPLE_CONVERSATION_PARAMS,
mistral_completion,
)
from vibe.core.types import (
AssistantEvent,
ToolCallEvent,
ToolResultEvent,
UserMessageEvent,
)
TODO_TOOL_CALL = [
{
"id": "call_todo_1",
"function": {"name": "todo", "arguments": '{"action": "read"}'},
"index": 0,
}
]
def _chunked_completion(reasoning: str, answer: str) -> dict[str, Any]:
completion = mistral_completion(answer)
completion["choices"][0]["message"]["content"] = [
{"type": "thinking", "thinking": [{"type": "text", "text": reasoning}]},
{"type": "text", "text": answer},
]
return completion
@pytest.mark.asyncio
async def test_act_completes_against_real_backend(mistral_api: MistralAPI) -> None:
# A plain prompt yields a user event then the backend's assistant reply, and
# the loop accumulates prompt+completion tokens from the usage block.
mistral_api.reply(
mistral_completion("Hi there!", prompt_tokens=120, completion_tokens=30)
)
agent = build_e2e_agent_loop()
events = [event async for event in agent.act("Hello")]
assert [type(e) for e in events] == [UserMessageEvent, AssistantEvent]
assert isinstance(events[1], AssistantEvent)
assert events[1].content == "Hi there!"
assert agent.stats.context_tokens == 150
@pytest.mark.asyncio
async def test_act_streaming(mistral_api: MistralAPI) -> None:
# Streamed SSE chunks are reassembled into the final assistant content.
_, chunks, _ = STREAMED_SIMPLE_CONVERSATION_PARAMS[0]
mistral_api.reply_stream(chunks)
agent = build_e2e_agent_loop(enable_streaming=True)
events = [event async for event in agent.act("Hi")]
assert assistant_text(events).endswith("Some content")
@pytest.mark.asyncio
async def test_act_tool_call_round_trip(mistral_api: MistralAPI) -> None:
# A tool call is parsed, executed, and its result fed back for a final reply.
mistral_api.reply(
mistral_completion("", tool_calls=TODO_TOOL_CALL),
mistral_completion("All done"),
)
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
events = [event async for event in agent.act("Show my todos")]
assert any(isinstance(e, ToolCallEvent) for e in events)
assert any(isinstance(e, ToolResultEvent) for e in events)
final = [e for e in events if isinstance(e, AssistantEvent)]
assert final and final[-1].content == "All done"
@pytest.mark.asyncio
async def test_act_serializes_tools_in_request_payload(mistral_api: MistralAPI) -> None:
# Enabled tools are serialized into the outgoing chat-completions request.
mistral_api.reply(mistral_completion("ok"))
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
_ = [event async for event in agent.act("Hello")]
names = {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
assert "todo" in names
@pytest.mark.asyncio
async def test_act_extracts_text_from_chunked_content_array(
mistral_api: MistralAPI,
) -> None:
# A thinking+text content array has its text part extracted as the reply.
mistral_api.reply(_chunked_completion("thinking hard", "the answer"))
agent = build_e2e_agent_loop()
events = [event async for event in agent.act("Why?")]
assistant = [e for e in events if isinstance(e, AssistantEvent)]
assert assistant and assistant[-1].content == "the answer"

View file

@ -0,0 +1,170 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, cast
from pydantic import BaseModel
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.tools.builtins.bash import BashResult
from vibe.core.tools.permissions import RequiredPermission
from vibe.core.types import ApprovalResponse, BaseEvent, ToolResultEvent
def _bash_call(command: str, timeout: int | None = None) -> dict[str, Any]:
arguments: dict[str, Any] = {"command": command}
if timeout is not None:
arguments["timeout"] = timeout
return mistral_completion(
"",
tool_calls=[
{
"id": "call_bash",
"function": {"name": "bash", "arguments": json.dumps(arguments)},
"index": 0,
}
],
)
async def _run_bash(
mistral_api: MistralAPI,
command: str,
*,
timeout: int | None = None,
agent_name: str = BuiltinAgentName.AUTO_APPROVE,
approval: ApprovalResponse | None = None,
) -> ToolResultEvent:
mistral_api.reply(_bash_call(command, timeout), mistral_completion("done"))
agent = build_e2e_agent_loop(
config=e2e_config(enabled_tools=["bash"]), agent_name=agent_name
)
if approval is not None:
async def approval_callback(
_tool_name: str,
_args: BaseModel,
_tool_call_id: str,
_rp: list[RequiredPermission] | None = None,
) -> tuple[ApprovalResponse, str | None]:
return (approval, None)
agent.set_approval_callback(approval_callback)
events: list[BaseEvent] = [event async for event in agent.act("go")]
return next(e for e in events if isinstance(e, ToolResultEvent))
@pytest.mark.asyncio
async def test_bash_captures_stdout(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "echo hello")
bash_result = cast(BashResult, result.result)
assert bash_result.returncode == 0
assert "hello" in bash_result.stdout
@pytest.mark.asyncio
async def test_bash_captures_stderr(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "echo oops >&2")
bash_result = cast(BashResult, result.result)
assert "oops" in bash_result.stderr
@pytest.mark.asyncio
async def test_bash_nonzero_exit_surfaces_as_error(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "exit 3")
assert result.error is not None
assert "Return code: 3" in result.error
@pytest.mark.asyncio
async def test_bash_timeout_surfaces_as_error(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "sleep 5", timeout=1)
assert result.error is not None
assert "timed out" in result.error.lower()
@pytest.mark.asyncio
async def test_bash_output_truncated_to_max_bytes(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "yes x | head -c 100000")
bash_result = cast(BashResult, result.result)
assert len(bash_result.stdout) <= 16_000
@pytest.mark.asyncio
async def test_bash_denylisted_command_is_skipped(mistral_api: MistralAPI) -> None:
result = await _run_bash(
mistral_api, "vim file.txt", agent_name=BuiltinAgentName.DEFAULT
)
assert result.skipped is True
assert result.skip_reason is not None
assert "denied" in result.skip_reason.lower()
@pytest.mark.asyncio
async def test_bash_allowlisted_command_runs_without_approval(
mistral_api: MistralAPI,
) -> None:
# No approval callback registered; an allowlisted command must run anyway.
result = await _run_bash(
mistral_api, "echo allowed", agent_name=BuiltinAgentName.DEFAULT
)
assert result.skipped is False
assert "allowed" in cast(BashResult, result.result).stdout
@pytest.mark.asyncio
async def test_bash_non_allowlisted_command_requires_approval(
mistral_api: MistralAPI,
) -> None:
result = await _run_bash(
mistral_api,
"touch newfile.txt",
agent_name=BuiltinAgentName.DEFAULT,
approval=ApprovalResponse.YES,
)
assert result.skipped is False
assert (Path.cwd() / "newfile.txt").exists()
@pytest.mark.asyncio
async def test_bash_non_allowlisted_command_denied_at_prompt_is_skipped(
mistral_api: MistralAPI,
) -> None:
result = await _run_bash(
mistral_api,
"touch denied.txt",
agent_name=BuiltinAgentName.DEFAULT,
approval=ApprovalResponse.NO,
)
assert result.skipped is True
assert not (Path.cwd() / "denied.txt").exists()
@pytest.mark.asyncio
async def test_bash_command_touching_outside_workdir_requires_approval(
mistral_api: MistralAPI, tmp_path: Path
) -> None:
outside = tmp_path / "outside.txt"
result = await _run_bash(
mistral_api,
f"touch {outside}",
agent_name=BuiltinAgentName.DEFAULT,
approval=ApprovalResponse.NO,
)
assert result.skipped is True
assert not outside.exists()

View file

@ -0,0 +1,136 @@
from __future__ import annotations
from typing import Any
import httpx
import pytest
import respx
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from tests.constants import CONNECTORS_BOOTSTRAP_PATH
from vibe.core.config import ConnectorConfig
def _connector(
*,
connector_id: str = "conn-1",
name: str = "wiki",
is_ready: bool = True,
tools: list[dict[str, Any]] | None = None,
auth_action: dict[str, Any] | None = None,
) -> dict[str, Any]:
return {
"id": connector_id,
"name": name,
"status": {"is_ready": is_ready},
"tools": tools or [],
"auth_action": auth_action,
}
def _tool(name: str = "search", description: str = "Search docs") -> dict[str, Any]:
return {
"name": name,
"description": description,
"inputSchema": {"type": "object", "properties": {"query": {"type": "string"}}},
}
def _set_bootstrap(router: respx.MockRouter, connectors: list[dict[str, Any]]) -> None:
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
return_value=httpx.Response(200, json={"connectors": connectors})
)
def _connectors_config(*aliases: str) -> Any:
return e2e_config(
connectors=[ConnectorConfig(name=alias, disabled=False) for alias in aliases]
)
async def _offered_tool_names(
mistral_api: MistralAPI, *enabled_connectors: str
) -> set[str]:
# Drive one turn and read which tools the AgentLoop serialized into the
# outgoing chat-completions request — the only connector-visible surface.
mistral_api.reply(mistral_completion("ok"))
agent = build_e2e_agent_loop(config=_connectors_config(*enabled_connectors))
_ = [event async for event in agent.act("hello")]
return {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
@pytest.mark.asyncio
async def test_connector_tools_are_offered_to_the_model(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(
mock_mistral, [_connector(name="wiki", tools=[_tool("search"), _tool("read")])]
)
names = await _offered_tool_names(mistral_api, "wiki")
assert "connector_wiki_search" in names
assert "connector_wiki_read" in names
@pytest.mark.asyncio
async def test_colliding_connector_aliases_are_disambiguated(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(
mock_mistral,
[
_connector(connector_id="c-1", name="mcp", tools=[_tool("a")]),
_connector(connector_id="c-2", name="mcp", tools=[_tool("b")]),
],
)
names = await _offered_tool_names(mistral_api, "mcp", "mcp_2")
assert "connector_mcp_a" in names
assert "connector_mcp_2_b" in names
@pytest.mark.asyncio
async def test_not_ready_connector_offers_no_tools(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(
mock_mistral,
[
_connector(
name="linear",
is_ready=False,
tools=[_tool("search")],
auth_action={"type": "oauth"},
)
],
)
names = await _offered_tool_names(mistral_api, "linear")
assert not any(name.startswith("connector_linear") for name in names)
@pytest.mark.asyncio
async def test_disabled_connector_tools_are_withheld(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(mock_mistral, [_connector(name="wiki", tools=[_tool("search")])])
# No ConnectorConfig entry → the connector stays disabled by default.
names = await _offered_tool_names(mistral_api)
assert "connector_wiki_search" not in names
@pytest.mark.asyncio
async def test_bootstrap_failure_still_lets_the_agent_run(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
mock_mistral.get(CONNECTORS_BOOTSTRAP_PATH).mock(return_value=httpx.Response(500))
names = await _offered_tool_names(mistral_api, "wiki")
assert not any(name.startswith("connector_") for name in names)

View file

@ -0,0 +1,144 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.tools.builtins.ask_user_question import (
AskUserQuestionArgs,
AskUserQuestionResult,
)
from vibe.core.types import PlanReviewEndedEvent, PlanReviewRequestedEvent, Role
from vibe.core.utils.tags import VIBE_WARNING_TAG
EXIT_PLAN_TOOL_CALL = [
{
"id": "call_exit_plan_1",
"function": {"name": "exit_plan_mode", "arguments": "{}"},
"index": 0,
}
]
def _user_message_ids(agent: Any) -> list[str]:
return [m.message_id for m in agent.messages if m.role == Role.user]
def _assistant_message_id(agent: Any) -> str:
return next(m.message_id for m in agent.messages if m.role == Role.assistant)
@pytest.mark.asyncio
async def test_fork_copies_all_non_system_messages(mistral_api: MistralAPI) -> None:
# fork() with no anchor clones every non-system message into the child loop.
mistral_api.reply(mistral_completion("Hi there!"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Hello")]
forked = await agent.fork()
non_system = [m for m in forked.messages if m.role != Role.system]
assert [m.role for m in non_system] == [Role.user, Role.assistant]
assert forked.parent_session_id == agent.session_id
assert forked.session_id != agent.session_id
@pytest.mark.asyncio
async def test_fork_from_message_id_truncates_at_next_user_turn(
mistral_api: MistralAPI,
) -> None:
# Forking from a user message keeps that turn but drops everything from the next one.
mistral_api.reply(mistral_completion("First"), mistral_completion("Second"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Turn one")]
_ = [event async for event in agent.act("Turn two")]
first_user_id = _user_message_ids(agent)[0]
forked = await agent.fork(first_user_id)
contents = [m.content for m in forked.messages if m.role != Role.system]
assert contents == ["Turn one", "First"]
@pytest.mark.asyncio
async def test_fork_from_unknown_message_id_raises(mistral_api: MistralAPI) -> None:
# An unknown anchor id is rejected rather than silently forking everything.
mistral_api.reply(mistral_completion("Hi"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Hello")]
with pytest.raises(ValueError, match="unknown message_id"):
await agent.fork("does-not-exist")
@pytest.mark.asyncio
async def test_fork_from_assistant_message_id_raises(mistral_api: MistralAPI) -> None:
# Forking is only allowed from user turns; an assistant anchor is rejected.
mistral_api.reply(mistral_completion("Hi"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Hello")]
assistant_id = _assistant_message_id(agent)
with pytest.raises(ValueError, match="only supported for user messages"):
await agent.fork(assistant_id)
def _plan_agent(mistral_api: MistralAPI) -> Any:
mistral_api.reply(
mistral_completion("", tool_calls=EXIT_PLAN_TOOL_CALL),
mistral_completion("Staying in plan mode."),
)
return build_e2e_agent_loop(
config=e2e_config(enabled_tools=["exit_plan_mode"]),
agent_name=BuiltinAgentName.PLAN,
)
@pytest.mark.asyncio
async def test_plan_mode_emits_review_requested_and_ended_events(
mistral_api: MistralAPI,
) -> None:
# An exit_plan_mode round trip brackets the tool with review-requested/ended events.
agent = _plan_agent(mistral_api)
async def stay_in_plan(_: AskUserQuestionArgs) -> AskUserQuestionResult:
return AskUserQuestionResult(cancelled=True, answers=[])
agent.set_user_input_callback(stay_in_plan)
events = [event async for event in agent.act("Make a plan")]
assert any(isinstance(e, PlanReviewRequestedEvent) for e in events)
assert any(isinstance(e, PlanReviewEndedEvent) for e in events)
@pytest.mark.asyncio
async def test_plan_mode_injects_updated_plan_when_file_changed(
mistral_api: MistralAPI,
) -> None:
# If the plan file changes during review, its new content is injected back as context.
agent = _plan_agent(mistral_api)
plan_path: Path | None = None
async def edit_plan_then_decline(_: AskUserQuestionArgs) -> AskUserQuestionResult:
assert plan_path is not None
plan_path.parent.mkdir(parents=True, exist_ok=True)
plan_path.write_text("# Updated plan\nStep 1")
return AskUserQuestionResult(cancelled=True, answers=[])
agent.set_user_input_callback(edit_plan_then_decline)
async for event in agent.act("Make a plan"):
if isinstance(event, PlanReviewRequestedEvent):
plan_path = event.file_path
injected = [m for m in agent.messages if getattr(m, "injected", False)]
assert any(
m.content and VIBE_WARNING_TAG in m.content and "# Updated plan" in m.content
for m in injected
)

View file

@ -0,0 +1,178 @@
from __future__ import annotations
import pytest
from tests.agent_loop.e2e.conftest import (
ProviderAPI,
anthropic_e2e_config,
assistant_text,
build_e2e_agent_loop,
openai_responses_e2e_config,
)
from tests.backend.data.anthropic import (
anthropic_message,
anthropic_reasoning_tool_use_stream,
anthropic_request_content_blocks,
anthropic_text_stream,
anthropic_tool_use,
)
from tests.backend.data.openai_responses import (
openai_function_call_item,
openai_message_item,
openai_reasoning_tool_call_stream,
openai_response,
openai_text_stream,
)
from vibe.core.types import ToolResultEvent
class TestAnthropic:
@pytest.mark.asyncio
async def test_agent_answers(self, anthropic_api: ProviderAPI) -> None:
# A plain prompt is answered through the Anthropic messages wire.
anthropic_api.reply(anthropic_message("pong"))
agent = build_e2e_agent_loop(config=anthropic_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 15
@pytest.mark.asyncio
async def test_agent_streams(self, anthropic_api: ProviderAPI) -> None:
# Anthropic SSE deltas are reassembled into the final assistant content.
anthropic_api.reply_stream(anthropic_text_stream("pong"))
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(), enable_streaming=True
)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_executes_tool_call(self, anthropic_api: ProviderAPI) -> None:
# A tool_use turn runs the tool, then Anthropic returns the final answer.
anthropic_api.reply(
anthropic_tool_use("todo", {"action": "read"}),
anthropic_message("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"])
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, anthropic_api: ProviderAPI
) -> None:
# A streamed thinking + tool_use turn runs the tool, then replays that
# reasoning/tool history into the follow-up Anthropic request.
anthropic_api.reply_streams(
anthropic_reasoning_tool_use_stream("todo", '{"action": "read"}'),
anthropic_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"]), enable_streaming=True
)
events = [event async for event in agent.act("What's on my todo list?")]
assert "Your list is empty." in assistant_text(events)
blocks = anthropic_request_content_blocks(anthropic_api.request_json)
thinking = [b["thinking"] for b in blocks if b.get("type") == "thinking"]
tool_uses = [b for b in blocks if b.get("type") == "tool_use"]
assert any("thinking..." in text for text in thinking)
assert tool_uses
class TestOpenAIResponses:
@pytest.mark.asyncio
async def test_agent_answers(self, openai_responses_api: ProviderAPI) -> None:
# A plain prompt is answered through the OpenAI Responses wire.
openai_responses_api.reply(openai_response([openai_message_item("pong")]))
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 12
@pytest.mark.asyncio
async def test_agent_streams(self, openai_responses_api: ProviderAPI) -> None:
# OpenAI Responses SSE deltas are reassembled into the final content.
openai_responses_api.reply_stream(openai_text_stream("pong"))
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(), enable_streaming=True
)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_captures_reasoning(
self, openai_responses_api: ProviderAPI
) -> None:
# Commentary phase output is captured as reasoning on the assistant message.
openai_responses_api.reply(
openai_response(
[
openai_message_item("Let me think.", phase="commentary"),
openai_message_item("pong", phase="final_answer"),
],
output_tokens=5,
)
)
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert any(
m.reasoning_content and "Let me think." in m.reasoning_content
for m in agent.messages
)
@pytest.mark.asyncio
async def test_agent_executes_tool_call(
self, openai_responses_api: ProviderAPI
) -> None:
# A function_call item runs the tool, then OpenAI returns the final answer.
openai_responses_api.reply(
openai_response([openai_function_call_item("todo", '{"action": "read"}')]),
openai_response([openai_message_item("Your list is empty.")]),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"])
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, openai_responses_api: ProviderAPI
) -> None:
# A streamed commentary + function_call turn runs the tool, then OpenAI
# streams the final answer on the follow-up request.
openai_responses_api.reply_streams(
openai_reasoning_tool_call_stream("todo", '{"action": "read"}'),
openai_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"]),
enable_streaming=True,
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)

View file

@ -0,0 +1,224 @@
from __future__ import annotations
from collections.abc import Iterator
from pathlib import Path
from git import Repo
import httpx
import pytest
import respx
from tests.agent_loop.e2e.conftest import build_e2e_agent_loop
from tests.constants import (
CONNECTORS_BOOTSTRAP_PATH,
MISTRAL_BASE_URL,
TELEPORT_COMPLETE_URL,
TELEPORT_SESSIONS_PATH,
)
from vibe.core.agent_loop import AgentLoop, TeleportError
from vibe.core.teleport.types import (
TeleportCheckingGitEvent,
TeleportCompleteEvent,
TeleportPushingEvent,
TeleportPushRequiredEvent,
TeleportPushResponseEvent,
TeleportStartingWorkflowEvent,
)
# vibe_code_sessions_base_url is an exclude=True field, so it is dropped when the
# agent manager re-derives config via model_dump(); the default url is unavoidable.
SESSIONS_BASE_URL = "https://chat.mistral.ai"
SESSIONS_URL = f"{SESSIONS_BASE_URL}{TELEPORT_SESSIONS_PATH}"
def _sessions_ok() -> httpx.Response:
return httpx.Response(
200,
json={
"sessionId": "controller-session-id",
"webSessionId": "web-session-id",
"projectId": "project-id",
"status": "running",
"url": TELEPORT_COMPLETE_URL,
},
)
def _commit(repo: Repo, message: str) -> str:
(Path(repo.working_dir) / "file.txt").write_text(f"{message}\n")
repo.index.add(["file.txt"])
repo.index.commit(message)
return repo.head.commit.hexsha
def _init_repo(workdir: Path) -> Repo:
# origin is a local bare repo so fetch/push stay offline and instant; a
# separate github-url remote satisfies Teleport's GitHub-only detection.
bare = Repo.init(workdir.with_name(f"{workdir.name}_origin.git"), bare=True)
repo = Repo.init(workdir, initial_branch="work")
repo.config_writer().set_value("user", "name", "Tester").release()
repo.config_writer().set_value("user", "email", "t@example.com").release()
repo.create_remote("origin", str(bare.git_dir))
repo.create_remote("hub", "https://github.com/owner/repo.git")
return repo
def _repo_with_pushed_branch(workdir: Path) -> Repo:
repo = _init_repo(workdir)
_commit(repo, "initial")
repo.git.push("origin", "work")
return repo
@pytest.fixture
def mock_sessions() -> Iterator[respx.MockRouter]:
# One router stubs both hosts: the connector bootstrap (api.mistral.ai, hit on
# agent init) and the teleport sessions endpoint (chat.mistral.ai).
with respx.mock(assert_all_called=False) as router:
router.get(f"{MISTRAL_BASE_URL}{CONNECTORS_BOOTSTRAP_PATH}").mock(
return_value=httpx.Response(200, json={"connectors": []})
)
router.post(SESSIONS_URL).mock(return_value=_sessions_ok())
yield router
async def _drain(
agent: AgentLoop, prompt: str | None, *, approve: bool | None = None
) -> list[object]:
gen = agent.teleport_to_vibe_code(prompt)
events: list[object] = []
response: TeleportPushResponseEvent | None = None
while True:
try:
event = await gen.asend(response)
except StopAsyncIteration:
break
events.append(event)
response = None
if isinstance(event, TeleportPushRequiredEvent) and approve is not None:
response = TeleportPushResponseEvent(approved=approve)
return events
@pytest.mark.asyncio
async def test_teleport_completes_when_branch_already_pushed(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
_repo_with_pushed_branch(tmp_working_directory)
events = await _drain(build_e2e_agent_loop(), "do the thing")
assert [type(e) for e in events] == [
TeleportCheckingGitEvent,
TeleportStartingWorkflowEvent,
TeleportCompleteEvent,
]
assert isinstance(events[-1], TeleportCompleteEvent)
assert events[-1].url == TELEPORT_COMPLETE_URL
@pytest.mark.asyncio
async def test_teleport_sends_repo_metadata_and_diff(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
commit = repo.head.commit.hexsha
(tmp_working_directory / "file.txt").write_text("uncommitted change\n")
await _drain(build_e2e_agent_loop(), "ship it")
payload = mock_sessions.calls.last.request.read().decode()
assert "https://github.com/owner/repo.git" in payload
assert "work" in payload
assert commit in payload
assert "zstd" in payload
@pytest.mark.asyncio
async def test_teleport_pushes_then_completes_when_approved(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
head = _commit(repo, "second")
events = await _drain(build_e2e_agent_loop(), "ship it", approve=True)
assert [type(e) for e in events] == [
TeleportCheckingGitEvent,
TeleportPushRequiredEvent,
TeleportPushingEvent,
TeleportStartingWorkflowEvent,
TeleportCompleteEvent,
]
assert repo.remote("origin").refs["work"].commit.hexsha == head
@pytest.mark.asyncio
async def test_teleport_aborts_when_push_declined(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
_commit(repo, "second")
with pytest.raises(TeleportError, match="not pushed"):
await _drain(build_e2e_agent_loop(), "ship it", approve=False)
assert not mock_sessions.post(SESSIONS_URL).called
@pytest.mark.asyncio
async def test_teleport_fails_when_push_fails(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
_commit(repo, "second")
repo.git.remote("set-url", "--push", "origin", "/nonexistent/repo.git")
with pytest.raises(TeleportError, match="Failed to push"):
await _drain(build_e2e_agent_loop(), "ship it", approve=True)
@pytest.mark.asyncio
async def test_teleport_requires_a_branch(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
repo.git.checkout(repo.head.commit.hexsha) # detach HEAD
with pytest.raises(TeleportError, match="checked-out branch"):
await _drain(build_e2e_agent_loop(), "ship it")
@pytest.mark.asyncio
async def test_teleport_rejects_empty_prompt(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _init_repo(tmp_working_directory)
_commit(repo, "initial")
with pytest.raises(TeleportError, match="non-empty prompt"):
await _drain(build_e2e_agent_loop(), None)
@pytest.mark.asyncio
async def test_teleport_surfaces_http_error(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
_repo_with_pushed_branch(tmp_working_directory)
mock_sessions.post(SESSIONS_URL).mock(return_value=httpx.Response(500, text="boom"))
with pytest.raises(TeleportError, match="start failed"):
await _drain(build_e2e_agent_loop(), "ship it")
@pytest.mark.asyncio
async def test_teleport_unsupported_without_github_remote(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = Repo.init(tmp_working_directory, initial_branch="work")
repo.config_writer().set_value("user", "name", "Tester").release()
repo.config_writer().set_value("user", "email", "t@example.com").release()
_commit(repo, "initial")
with pytest.raises(TeleportError, match="GitHub"):
await _drain(build_e2e_agent_loop(), "ship it")

View file

@ -0,0 +1,93 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, cast
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from vibe.core.tools.builtins.grep import GrepResult
from vibe.core.tools.builtins.read import ReadResult
from vibe.core.tools.builtins.todo import TodoResult
from vibe.core.types import BaseEvent, ToolResultEvent
def _tool_call(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
return mistral_completion(
"",
tool_calls=[
{
"id": f"call_{name}",
"function": {"name": name, "arguments": json.dumps(arguments)},
"index": 0,
}
],
)
async def _run_tool(
mistral_api: MistralAPI, name: str, arguments: dict[str, Any]
) -> ToolResultEvent:
# Drive one tool call against the real cwd, then a final reply, and return
# the single tool result for the test to assert on.
mistral_api.reply(_tool_call(name, arguments), mistral_completion("done"))
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=[name]))
events: list[BaseEvent] = [event async for event in agent.act("go")]
result = next(e for e in events if isinstance(e, ToolResultEvent))
assert result.error is None
return result
@pytest.mark.asyncio
async def test_write_file_tool_creates_file(mistral_api: MistralAPI) -> None:
target = Path.cwd() / "note.txt"
await _run_tool(mistral_api, "write_file", {"path": str(target), "content": "hi\n"})
assert target.read_text() == "hi\n"
@pytest.mark.asyncio
async def test_read_tool_returns_file_content(mistral_api: MistralAPI) -> None:
target = Path.cwd() / "note.txt"
target.write_text("hello world\n")
result = await _run_tool(mistral_api, "read", {"file_path": str(target)})
assert "hello world" in cast(ReadResult, result.result).content
@pytest.mark.asyncio
async def test_edit_tool_replaces_text(mistral_api: MistralAPI) -> None:
target = Path.cwd() / "note.txt"
target.write_text("hello world\n")
await _run_tool(
mistral_api,
"edit",
{"file_path": str(target), "old_string": "hello", "new_string": "goodbye"},
)
assert target.read_text() == "goodbye world\n"
@pytest.mark.asyncio
async def test_grep_tool_finds_matches(mistral_api: MistralAPI) -> None:
(Path.cwd() / "note.txt").write_text("needle here\n")
result = await _run_tool(mistral_api, "grep", {"pattern": "needle", "path": "."})
assert cast(GrepResult, result.result).match_count >= 1
@pytest.mark.asyncio
async def test_todo_tool_writes_items(mistral_api: MistralAPI) -> None:
result = await _run_tool(
mistral_api,
"todo",
{"action": "write", "todos": [{"id": "1", "content": "ship it"}]},
)
assert cast(TodoResult, result.result).total_count == 1

View file

@ -0,0 +1,439 @@
from __future__ import annotations
import asyncio
from pathlib import Path
from typing import cast
from unittest.mock import AsyncMock, patch
import pytest
from tests.conftest import (
build_test_agent_loop,
build_test_vibe_config,
make_test_models,
)
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agent_loop import CompactionFailedError
from vibe.core.config import ModelConfig
from vibe.core.types import (
AssistantEvent,
CompactEndEvent,
CompactStartEvent,
FunctionCall,
LLMMessage,
Role,
ToolCall,
UserMessageEvent,
)
def _get_auto_compact_properties(
telemetry_events: list[dict[str, object]],
) -> dict[str, object]:
auto_compact = [
event
for event in telemetry_events
if event.get("event_name") == "vibe.auto_compact_triggered"
]
assert len(auto_compact) == 1
return cast(dict[str, object], auto_compact[0]["properties"])
@pytest.mark.asyncio
async def test_auto_compact_emits_correct_events(telemetry_events: list[dict]) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.stats.context_tokens = 2
old_session_id = agent.session_id
events = [ev async for ev in agent.act("Hello")]
assert len(events) == 4
assert isinstance(events[0], UserMessageEvent)
assert isinstance(events[1], CompactStartEvent)
assert isinstance(events[2], CompactEndEvent)
assert isinstance(events[3], AssistantEvent)
start: CompactStartEvent = events[1]
end: CompactEndEvent = events[2]
final: AssistantEvent = events[3]
assert start.current_context_tokens == 2
assert start.threshold == 1
assert isinstance(end, CompactEndEvent)
assert final.content == "<final>"
properties = _get_auto_compact_properties(telemetry_events)
assert properties["nb_context_tokens_before"] == 2
assert properties["auto_compact_threshold"] == 1
assert properties["status"] == "success"
assert properties["session_id"] == old_session_id
assert properties["parent_session_id"] is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
("side_effect", "expected_exception", "match", "expected_status"),
[
pytest.param(
RuntimeError("boom"), RuntimeError, "boom", "failure", id="failure"
),
pytest.param(
asyncio.CancelledError(),
asyncio.CancelledError,
None,
"cancelled",
id="cancelled",
),
],
)
async def test_auto_compact_emits_terminal_telemetry(
side_effect: BaseException,
expected_exception: type[BaseException],
match: str | None,
expected_status: str,
telemetry_events: list[dict],
) -> None:
backend = FakeBackend([[mock_llm_chunk(content="<final>")]])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.stats.context_tokens = 2
old_session_id = agent.session_id
events = []
with patch.object(agent, "compact", AsyncMock(side_effect=side_effect)):
if match is None:
with pytest.raises(expected_exception):
async for event in agent.act("Hello"):
events.append(event)
else:
with pytest.raises(expected_exception, match=match):
async for event in agent.act("Hello"):
events.append(event)
assert len(events) == 2
assert isinstance(events[0], UserMessageEvent)
assert isinstance(events[1], CompactStartEvent)
properties = _get_auto_compact_properties(telemetry_events)
assert properties["nb_context_tokens_before"] == 2
assert properties["auto_compact_threshold"] == 1
assert properties["status"] == expected_status
assert properties["session_id"] == old_session_id
assert properties["parent_session_id"] is None
@pytest.mark.asyncio
async def test_auto_compact_observer_sees_user_msg_not_summary() -> None:
"""Observer sees the original user message and final response.
Compact internals (summary request, LLM summary) are invisible
to the observer because they happen inside silent() / reset().
"""
observed: list[tuple[Role, str | None]] = []
def observer(msg: LLMMessage) -> None:
observed.append((msg.role, msg.content))
backend = FakeBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(
config=cfg, message_observer=observer, backend=backend
)
agent.stats.context_tokens = 2
[_ async for _ in agent.act("Hello")]
roles = [r for r, _ in observed]
assert roles == [Role.system, Role.user, Role.assistant]
assert observed[1][1] == "Hello"
assert observed[2][1] == "<final>"
@pytest.mark.asyncio
async def test_auto_compact_observer_does_not_see_summary_request() -> None:
"""The compact summary request and LLM response must not leak to observer."""
observed: list[tuple[Role, str | None]] = []
def observer(msg: LLMMessage) -> None:
observed.append((msg.role, msg.content))
backend = FakeBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(
config=cfg, message_observer=observer, backend=backend
)
agent.stats.context_tokens = 2
[_ async for _ in agent.act("Hello")]
contents = [c for _, c in observed]
assert "<summary>" not in contents
assert all("compact" not in (c or "").lower() for c in contents)
@pytest.mark.asyncio
async def test_compact_replaces_messages_with_context() -> None:
backend = FakeBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.stats.context_tokens = 2
[_ async for _ in agent.act("Hello")]
# After compact + final response: system, compaction context, final.
assert agent.messages[0].role == Role.system
assert agent.messages[-1].role == Role.assistant
assert agent.messages[-1].content == "<final>"
class _ModelTrackingBackend(FakeBackend):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.requested_models: list[ModelConfig] = []
async def complete(self, *, model, **kwargs):
self.requested_models.append(model)
return await super().complete(model=model, **kwargs)
@pytest.mark.asyncio
async def test_compact_uses_compaction_model() -> None:
"""When compaction_model is set, compact() uses it instead of active_model."""
compaction = ModelConfig(
name="compaction-model",
provider="mistral",
alias="compaction",
auto_compact_threshold=1,
)
backend = _ModelTrackingBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(
models=make_test_models(auto_compact_threshold=1), compaction_model=compaction
)
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.stats.context_tokens = 2
[_ async for _ in agent.act("Hello")]
assert backend.requested_models[0].name == "compaction-model"
assert backend.requested_models[1].name != "compaction-model"
@pytest.mark.asyncio
async def test_compact_uses_active_model_when_no_compaction_model() -> None:
"""Without compaction_model, compact() falls back to the active model."""
backend = _ModelTrackingBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.stats.context_tokens = 2
[_ async for _ in agent.act("Hello")]
active = cfg.get_active_model()
assert backend.requested_models[0].name == active.name
assert backend.requested_models[1].name == active.name
@pytest.mark.asyncio
async def test_compact_appends_extra_instructions_to_prompt() -> None:
backend = FakeBackend([[mock_llm_chunk(content="<summary>")]])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
agent.stats.context_tokens = 100
await agent.compact(extra_instructions="focus on auth")
compaction_prompt = backend.requests_messages[0][-1].content
assert compaction_prompt is not None
assert "## Additional Instructions" in compaction_prompt
assert "focus on auth" in compaction_prompt
@pytest.mark.asyncio
async def test_compact_uses_configured_compaction_prompt(
mock_prompts_dirs: tuple[Path, Path],
) -> None:
project_prompts, _ = mock_prompts_dirs
(project_prompts / "theorem_compact.md").write_text("Summarize theorem progress")
backend = FakeBackend([[mock_llm_chunk(content="<summary>")]])
cfg = build_test_vibe_config(
models=make_test_models(auto_compact_threshold=999),
compaction_prompt_id="theorem_compact",
)
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
agent.stats.context_tokens = 100
await agent.compact()
compaction_prompt = backend.requests_messages[0][-1].content
assert compaction_prompt == "Summarize theorem progress"
@pytest.mark.asyncio
async def test_compact_without_extra_instructions_has_no_additional_section() -> None:
backend = FakeBackend([[mock_llm_chunk(content="<summary>")]])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
agent.stats.context_tokens = 100
await agent.compact()
compaction_prompt = backend.requests_messages[0][-1].content
assert compaction_prompt is not None
assert "## Additional Instructions" not in compaction_prompt
@pytest.mark.asyncio
async def test_compact_raises_on_tool_call_when_flag_enabled() -> None:
"""With the flag on, a compaction that returns a tool call raises."""
backend = FakeBackend([
[
mock_llm_chunk(
content="",
tool_calls=[
ToolCall(
id="t1",
index=0,
function=FunctionCall(name="bash", arguments="{}"),
)
],
)
]
])
cfg = build_test_vibe_config(
models=make_test_models(auto_compact_threshold=999),
raise_on_compaction_failure=True,
)
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
agent.stats.context_tokens = 100
with pytest.raises(CompactionFailedError) as exc_info:
await agent.compact()
assert exc_info.value.reason == "tool_call"
@pytest.mark.asyncio
async def test_compact_raises_on_empty_summary_when_flag_enabled() -> None:
"""With the flag on, a compaction with empty content raises."""
backend = FakeBackend([[mock_llm_chunk(content=" ")]])
cfg = build_test_vibe_config(
models=make_test_models(auto_compact_threshold=999),
raise_on_compaction_failure=True,
)
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
agent.stats.context_tokens = 100
with pytest.raises(CompactionFailedError) as exc_info:
await agent.compact()
assert exc_info.value.reason == "empty_summary"
@pytest.mark.asyncio
async def test_compact_falls_back_when_flag_disabled() -> None:
"""With the flag off (default), empty content uses the legacy fallback."""
backend = FakeBackend([[mock_llm_chunk(content="")]])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="Hello"))
agent.stats.context_tokens = 100
summary = await agent.compact()
assert summary == "(no summary available)"
@pytest.mark.asyncio
async def test_compact_message_shape_preserves_prior_user_messages() -> None:
from vibe.core.compaction import parse_previous_user_messages
from vibe.core.prompts import UtilityPrompt
summary_prefix = UtilityPrompt.COMPACT_SUMMARY_PREFIX.read()
backend = FakeBackend([[mock_llm_chunk(content="fresh summary body")]])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
agent = build_test_agent_loop(config=cfg, backend=backend)
system_message_before = agent.messages[0]
agent.messages.append(LLMMessage(role=Role.user, content="first real ask"))
agent.messages.append(
LLMMessage(role=Role.user, content="middleware ping", injected=True)
)
agent.messages.append(LLMMessage(role=Role.assistant, content="ack"))
agent.messages.append(
LLMMessage(
role=Role.user,
content=f"{summary_prefix}\nprior summary blob",
injected=True,
)
)
agent.messages.append(LLMMessage(role=Role.user, content="follow-up ask"))
agent.stats.context_tokens = 100
await agent.compact()
final = list(agent.messages)
assert len(final) == 2 # [system, compaction_context]
assert final[0] is system_message_before
assert final[1].role == Role.user
assert final[1].injected is True
assert parse_previous_user_messages(final[1].content or "") == [
"first real ask",
"follow-up ask",
]
assert "Here are some of the most recent previous user messages" in (
final[1].content or ""
)
assert "<compaction_summary>" in (final[1].content or "")
assert "fresh summary body" in (final[1].content or "")
# Injected and prior-summary user messages must be filtered out.
assert all("middleware ping" not in (m.content or "") for m in final)
assert sum("prior summary blob" in (m.content or "") for m in final) == 0
@pytest.mark.asyncio
async def test_compact_preserves_user_messages_across_repeated_compactions() -> None:
from vibe.core.compaction import parse_previous_user_messages
backend = FakeBackend([
[mock_llm_chunk(content="summary one")],
[mock_llm_chunk(content="summary two")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999))
agent = build_test_agent_loop(config=cfg, backend=backend)
agent.messages.append(LLMMessage(role=Role.user, content="first ask"))
agent.stats.context_tokens = 100
await agent.compact()
agent.messages.append(LLMMessage(role=Role.user, content="second ask"))
agent.stats.context_tokens = 100
await agent.compact()
final = list(agent.messages)
assert len(final) == 2
assert parse_previous_user_messages(final[1].content or "") == [
"first ask",
"second ask",
]

View file

@ -0,0 +1,83 @@
from __future__ import annotations
from pathlib import Path
import pytest
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agent_loop import AgentLoop
from vibe.core.config import SessionLoggingConfig
from vibe.core.types import BaseEvent, SessionTitleUpdatedEvent, UserMessageEvent
def _make_agent_loop(tmp_path: Path) -> AgentLoop:
session_logging = SessionLoggingConfig(
save_dir=str(tmp_path / "sessions"), session_prefix="session", enabled=True
)
config = build_test_vibe_config(session_logging=session_logging)
backend = FakeBackend([
[mock_llm_chunk(content="ok")],
[mock_llm_chunk(content="ok")],
])
return build_test_agent_loop(config=config, backend=backend)
async def _collect(loop: AgentLoop, prompt: str, **kwargs) -> list[BaseEvent]:
return [ev async for ev in loop.act(prompt, **kwargs)]
class TestAgentLoopAutoTitleEvent:
@pytest.mark.asyncio
async def test_emits_event_on_first_user_message(self, tmp_path: Path) -> None:
loop = _make_agent_loop(tmp_path)
events = await _collect(loop, "rendered prompt", auto_title="Pretty title")
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
assert len(title_events) == 1
assert title_events[0].title == "Pretty title"
@pytest.mark.asyncio
async def test_event_fires_after_user_message_event(self, tmp_path: Path) -> None:
loop = _make_agent_loop(tmp_path)
events = await _collect(loop, "rendered", auto_title="Pretty")
indices = {
type(e).__name__: i
for i, e in enumerate(events)
if isinstance(e, (UserMessageEvent, SessionTitleUpdatedEvent))
}
assert indices["UserMessageEvent"] < indices["SessionTitleUpdatedEvent"]
@pytest.mark.asyncio
async def test_no_event_on_second_message(self, tmp_path: Path) -> None:
loop = _make_agent_loop(tmp_path)
await _collect(loop, "first", auto_title="First title")
events = await _collect(loop, "second", auto_title="Second title")
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
assert title_events == []
@pytest.mark.asyncio
async def test_no_event_when_auto_title_is_none(self, tmp_path: Path) -> None:
loop = _make_agent_loop(tmp_path)
events = await _collect(loop, "rendered", auto_title=None)
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
assert title_events == []
@pytest.mark.asyncio
async def test_no_event_when_session_logging_disabled(self, tmp_path: Path) -> None:
config = build_test_vibe_config()
backend = FakeBackend(mock_llm_chunk(content="ok"))
loop = build_test_agent_loop(config=config, backend=backend)
events = await _collect(loop, "rendered", auto_title="Pretty title")
title_events = [e for e in events if isinstance(e, SessionTitleUpdatedEvent)]
assert title_events == []

View file

@ -0,0 +1,505 @@
from __future__ import annotations
from unittest.mock import MagicMock
from mcp.types import (
CreateMessageRequestParams,
CreateMessageResult,
SamplingMessage,
TextContent,
)
import pytest
from tests.conftest import (
build_test_agent_loop,
build_test_vibe_config,
make_test_models,
)
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.telemetry.types import EntrypointMetadata
from vibe.core.tools.base import BaseToolConfig, ToolPermission
from vibe.core.types import (
Backend,
FunctionCall,
LLMChunk,
LLMMessage,
LLMUsage,
RefusalError,
Role,
StopInfo,
ToolCall,
)
def _two_model_vibe_config(active_model: str) -> VibeConfig:
"""VibeConfig with two models so we can switch active_model."""
models = [
ModelConfig(
name="mistral-vibe-cli-latest", provider="mistral", alias="devstral-latest"
),
ModelConfig(
name="devstral-small-latest", provider="mistral", alias="devstral-small"
),
]
providers = [
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
]
return build_test_vibe_config(
active_model=active_model, models=models, providers=providers
)
def _make_sampling_params() -> CreateMessageRequestParams:
return CreateMessageRequestParams(
messages=[
SamplingMessage(role="user", content=TextContent(type="text", text="Hi"))
],
systemPrompt=None,
temperature=None,
maxTokens=100,
)
@pytest.mark.asyncio
async def test_passes_x_affinity_header_when_asking_an_answer(vibe_config: VibeConfig):
backend = FakeBackend([mock_llm_chunk(content="Response")])
agent = build_test_agent_loop(config=vibe_config, backend=backend)
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_extra_headers) > 0
headers = backend.requests_extra_headers[0]
assert headers is not None
assert "x-affinity" in headers
assert headers["x-affinity"] == agent.session_id
@pytest.mark.asyncio
async def test_passes_x_affinity_header_when_asking_an_answer_streaming(
vibe_config: VibeConfig,
):
backend = FakeBackend([mock_llm_chunk(content="Response")])
agent = build_test_agent_loop(
config=vibe_config, backend=backend, enable_streaming=True
)
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_extra_headers) > 0
headers = backend.requests_extra_headers[0]
assert headers is not None
assert "x-affinity" in headers
assert headers["x-affinity"] == agent.session_id
@pytest.mark.asyncio
async def test_updates_tokens_stats_based_on_backend_response(vibe_config: VibeConfig):
chunk = mock_llm_chunk(content="Response", prompt_tokens=100, completion_tokens=50)
backend = FakeBackend([chunk])
agent = build_test_agent_loop(config=vibe_config, backend=backend)
[_ async for _ in agent.act("Hello")]
assert agent.stats.context_tokens == 150
@pytest.mark.asyncio
async def test_updates_tokens_stats_based_on_backend_response_streaming(
vibe_config: VibeConfig,
):
final_chunk = mock_llm_chunk(
content="Complete", prompt_tokens=200, completion_tokens=75
)
backend = FakeBackend([final_chunk])
agent = build_test_agent_loop(
config=vibe_config, backend=backend, enable_streaming=True
)
[_ async for _ in agent.act("Hello")]
assert agent.stats.context_tokens == 275
@pytest.mark.asyncio
async def test_passes_session_id_to_backend(vibe_config: VibeConfig):
backend = FakeBackend([mock_llm_chunk(content="Response")])
agent = build_test_agent_loop(config=vibe_config, backend=backend)
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_metadata) > 0
meta = backend.requests_metadata[0]
assert meta is not None
assert meta["session_id"] == agent.session_id
assert "parent_session_id" not in meta
assert "message_id" in meta
assert meta["call_type"] == "main_call"
assert meta["call_source"] == "vibe_code"
@pytest.mark.asyncio
async def test_passes_parent_session_id_to_backend_after_reset(vibe_config: VibeConfig):
backend = FakeBackend([
[mock_llm_chunk(content="Response")],
[mock_llm_chunk(content="Response after reset")],
])
agent = build_test_agent_loop(config=vibe_config, backend=backend)
[_ async for _ in agent.act("Hello")]
first_session_id = agent.session_id
await agent._reset_session()
[_ async for _ in agent.act("Hello again")]
assert len(backend.requests_metadata) >= 2
reset_meta = backend.requests_metadata[1]
assert reset_meta is not None
assert reset_meta["session_id"] == agent.session_id
assert reset_meta["parent_session_id"] == first_session_id
@pytest.mark.asyncio
async def test_passes_entrypoint_metadata_to_backend(vibe_config: VibeConfig):
metadata = EntrypointMetadata(
agent_entrypoint="acp",
agent_version="2.0.0",
client_name="vibe_ide",
client_version="0.5.0",
)
backend = FakeBackend([mock_llm_chunk(content="Response")])
agent = build_test_agent_loop(
config=vibe_config,
backend=backend,
enable_streaming=True,
entrypoint_metadata=metadata,
)
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_metadata) > 0
meta = backend.requests_metadata[0]
assert meta is not None
assert meta["agent_entrypoint"] == "acp"
assert meta["agent_version"] == "2.0.0"
assert meta["client_name"] == "vibe_ide"
assert meta["client_version"] == "0.5.0"
assert meta["session_id"] == agent.session_id
assert "message_id" in meta
assert meta["call_type"] == "main_call"
assert meta["call_source"] == "vibe_code"
@pytest.mark.asyncio
async def test_mcp_sampling_handler_uses_updated_backend_when_agent_backend_changes():
"""AgentLoop's MCP sampling handler uses current backend when backend is reassigned."""
backend1 = FakeBackend([mock_llm_chunk(content="from-backend-1")])
backend2 = FakeBackend([mock_llm_chunk(content="from-backend-2")])
config = _two_model_vibe_config("devstral-latest")
agent = build_test_agent_loop(config=config, backend=backend1)
handler = agent._sampling_handler
params = _make_sampling_params()
context = MagicMock()
result1 = await handler(context, params)
assert isinstance(result1, CreateMessageResult)
assert result1.content.type == "text"
assert result1.content.text == "from-backend-1"
assert len(backend1.requests_messages) == 1
assert len(backend2.requests_messages) == 0
agent.backend = backend2
result2 = await handler(context, params)
assert isinstance(result2, CreateMessageResult)
assert result2.content.type == "text"
assert result2.content.text == "from-backend-2"
assert len(backend1.requests_messages) == 1
assert len(backend2.requests_messages) == 1
@pytest.mark.asyncio
async def test_mcp_sampling_handler_uses_updated_config_when_agent_config_changes():
chunk = mock_llm_chunk(content="ok")
backend = FakeBackend([chunk])
config1 = _two_model_vibe_config("devstral-latest")
config2 = _two_model_vibe_config("devstral-small")
agent = build_test_agent_loop(config=config1, backend=backend)
handler = agent._sampling_handler
params = _make_sampling_params()
context = MagicMock()
result1 = await handler(context, params)
assert isinstance(result1, CreateMessageResult)
assert result1.model == "mistral-vibe-cli-latest"
agent._base_config = config2
agent.agent_manager.invalidate_config()
result2 = await handler(context, params)
assert isinstance(result2, CreateMessageResult)
assert result2.model == "devstral-small-latest"
@pytest.mark.asyncio
async def test_mcp_sampling_handler_sends_secondary_call_telemetry_metadata():
metadata = EntrypointMetadata(
agent_entrypoint="acp",
agent_version="2.0.0",
client_name="vibe_ide",
client_version="0.5.0",
)
backend = FakeBackend([
[mock_llm_chunk(content="Response")],
[mock_llm_chunk(content="Sampled response")],
])
agent = build_test_agent_loop(
config=_two_model_vibe_config("devstral-latest"),
backend=backend,
entrypoint_metadata=metadata,
)
[_ async for _ in agent.act("Hello")]
agent.parent_session_id = "parent-session-456"
result = await agent._sampling_handler(MagicMock(), _make_sampling_params())
assert isinstance(result, CreateMessageResult)
assert len(backend.requests_metadata) == 2
sampling_metadata = backend.requests_metadata[1]
assert sampling_metadata is not None
assert sampling_metadata["agent_entrypoint"] == "acp"
assert sampling_metadata["agent_version"] == "2.0.0"
assert sampling_metadata["client_name"] == "vibe_ide"
assert sampling_metadata["client_version"] == "0.5.0"
assert sampling_metadata["session_id"] == agent.session_id
assert sampling_metadata["parent_session_id"] == "parent-session-456"
assert sampling_metadata["message_id"] == next(
message.message_id for message in agent.messages if message.role == Role.user
)
assert sampling_metadata["call_type"] == "secondary_call"
assert sampling_metadata["call_source"] == "vibe_code"
assert len(backend.requests_extra_headers) == 2
sampling_headers = backend.requests_extra_headers[1]
assert sampling_headers is not None
assert sampling_headers["x-affinity"] == agent.session_id
def _generic_provider_vibe_config() -> VibeConfig:
"""VibeConfig with generic backend so no metadata header is sent."""
providers = [
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.GENERIC,
)
]
return build_test_vibe_config(providers=providers)
@pytest.mark.asyncio
async def test_mistral_metadata_header_call_type_per_turn() -> None:
"""First LLM call in a turn is main_call; second call (after tools) is secondary_call."""
tool_call = ToolCall(
id="call_1",
index=0,
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
)
backend = FakeBackend([
[mock_llm_chunk(content="Checking todos.", tool_calls=[tool_call])],
[mock_llm_chunk(content="Here are your todos.")],
])
config = build_test_vibe_config(
providers=[
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
],
enabled_tools=["todo"],
tools={"todo": BaseToolConfig(permission=ToolPermission.ALWAYS)},
)
agent = build_test_agent_loop(
config=config, backend=backend, agent_name=BuiltinAgentName.AUTO_APPROVE
)
[_ async for _ in agent.act("What's on my todo list?")]
assert len(backend.requests_metadata) == 2
first_metadata = backend.requests_metadata[0]
second_metadata = backend.requests_metadata[1]
assert first_metadata is not None
assert second_metadata is not None
assert first_metadata["call_type"] == "main_call"
assert second_metadata["call_type"] == "secondary_call"
@pytest.mark.asyncio
async def test_auto_compact_emits_summary_and_next_turn_metadata() -> None:
"""Compact emits summary then user-turn backend metadata in order."""
backend = FakeBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
config = build_test_vibe_config(
models=make_test_models(auto_compact_threshold=1),
providers=[
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
],
)
agent = build_test_agent_loop(config=config, backend=backend)
agent.stats.context_tokens = 2
original_session_id = agent.session_id
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_metadata) == 2
assert len(backend.requests_extra_headers) == 2
compact_metadata = backend.requests_metadata[0]
user_turn_metadata = backend.requests_metadata[1]
assert compact_metadata is not None
assert user_turn_metadata is not None
assert compact_metadata["call_type"] == "secondary_call"
assert compact_metadata["session_id"] == original_session_id
assert "parent_session_id" not in compact_metadata
assert user_turn_metadata["call_type"] == "main_call"
assert user_turn_metadata["session_id"] == agent.session_id
assert user_turn_metadata["parent_session_id"] == original_session_id
compact_headers = backend.requests_extra_headers[0]
user_turn_headers = backend.requests_extra_headers[1]
assert compact_headers is not None
assert user_turn_headers is not None
assert compact_headers["x-affinity"] == original_session_id
assert user_turn_headers["x-affinity"] == agent.session_id
@pytest.mark.asyncio
async def test_generic_provider_has_no_metadata_header() -> None:
"""Non-Mistral provider does not send the metadata header."""
backend = FakeBackend([mock_llm_chunk(content="Response")])
config = _generic_provider_vibe_config()
agent = build_test_agent_loop(config=config, backend=backend)
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_extra_headers) == 1
headers = backend.requests_extra_headers[0]
assert headers is not None
assert "metadata" not in headers
@pytest.mark.asyncio
async def test_provider_extra_headers_are_forwarded() -> None:
backend = FakeBackend([mock_llm_chunk(content="Response")])
providers = [
ProviderConfig(
name="custom",
api_base="https://custom.example.com/v1",
extra_headers={"X-Custom-Auth": "token123", "X-Org-Id": "org-456"},
)
]
models = [ModelConfig(name="test-model", provider="custom", alias="test")]
config = build_test_vibe_config(
active_model="test", models=models, providers=providers
)
agent = build_test_agent_loop(config=config, backend=backend)
[_ async for _ in agent.act("Hello")]
assert len(backend.requests_extra_headers) == 1
headers = backend.requests_extra_headers[0]
assert headers is not None
assert headers["X-Custom-Auth"] == "token123"
assert headers["X-Org-Id"] == "org-456"
def _refusal_chunk() -> LLMChunk:
return LLMChunk(
message=LLMMessage(role=Role.assistant, content=""),
usage=LLMUsage(prompt_tokens=10, completion_tokens=2),
stop=StopInfo(reason="refusal"),
)
@pytest.mark.asyncio
async def test_refusal_stop_reason_raises_refusal_error(vibe_config: VibeConfig):
backend = FakeBackend([_refusal_chunk()])
agent = build_test_agent_loop(config=vibe_config, backend=backend)
with pytest.raises(RefusalError):
[_ async for _ in agent.act("Hello")]
@pytest.mark.asyncio
async def test_refusal_stop_reason_raises_refusal_error_streaming(
vibe_config: VibeConfig,
):
backend = FakeBackend([_refusal_chunk()])
agent = build_test_agent_loop(
config=vibe_config, backend=backend, enable_streaming=True
)
with pytest.raises(RefusalError):
[_ async for _ in agent.act("Hello")]
def _refusal_chunk_with_details() -> LLMChunk:
return LLMChunk(
message=LLMMessage(role=Role.assistant, content=""),
usage=LLMUsage(prompt_tokens=10, completion_tokens=2),
stop=StopInfo(
reason="refusal",
category="cyber",
explanation="This request was declined for safety reasons.",
),
)
@pytest.mark.asyncio
async def test_refusal_error_carries_category_and_explanation(vibe_config: VibeConfig):
backend = FakeBackend([_refusal_chunk_with_details()])
agent = build_test_agent_loop(config=vibe_config, backend=backend)
with pytest.raises(RefusalError) as exc_info:
[_ async for _ in agent.act("Hello")]
err = exc_info.value
assert err.category == "cyber"
assert err.explanation == "This request was declined for safety reasons."
assert "This request was declined for safety reasons." in str(err)
assert "cyber" in str(err)
@pytest.mark.asyncio
async def test_refusal_error_carries_category_and_explanation_streaming(
vibe_config: VibeConfig,
):
backend = FakeBackend([_refusal_chunk_with_details()])
agent = build_test_agent_loop(
config=vibe_config, backend=backend, enable_streaming=True
)
with pytest.raises(RefusalError) as exc_info:
[_ async for _ in agent.act("Hello")]
err = exc_info.value
assert err.category == "cyber"
assert err.explanation == "This request was declined for safety reasons."
assert "This request was declined for safety reasons." in str(err)
assert "cyber" in str(err)

View file

@ -0,0 +1,195 @@
from __future__ import annotations
from pathlib import Path
import pytest
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agent_loop import ImagesNotSupportedError
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.types import Backend, FileImageSource, ImageAttachment, LLMMessage, Role
PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16
def _config_with_vision_flag(*, supports_images: bool) -> VibeConfig:
models = [
ModelConfig(
name="mistral-vibe-cli-latest",
provider="mistral",
alias="devstral-latest",
supports_images=supports_images,
)
]
providers = [
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
]
return build_test_vibe_config(
active_model="devstral-latest", models=models, providers=providers
)
def _config_with_both_models() -> VibeConfig:
models = [
ModelConfig(
name="vision-model",
provider="mistral",
alias="vision-alias",
supports_images=True,
),
ModelConfig(
name="text-model",
provider="mistral",
alias="text-alias",
supports_images=False,
),
]
providers = [
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
]
return build_test_vibe_config(
active_model="vision-alias", models=models, providers=providers
)
@pytest.fixture()
def png_attachment(tmp_path: Path) -> ImageAttachment:
p = tmp_path / "x.png"
p.write_bytes(PNG_BYTES)
return ImageAttachment(
source=FileImageSource(path=p), alias="x.png", mime_type="image/png"
)
@pytest.mark.asyncio
async def test_act_raises_when_model_lacks_vision(
png_attachment: ImageAttachment,
) -> None:
config = _config_with_vision_flag(supports_images=False)
backend = FakeBackend([mock_llm_chunk(content="ok")])
agent = build_test_agent_loop(config=config, backend=backend)
initial_message_count = len(agent.messages)
with pytest.raises(ImagesNotSupportedError):
async for _ in agent.act("look", images=[png_attachment]):
pass
assert backend.requests_extra_headers == [] # no LLM call was made
# Capability check runs *before* checkpoint creation and history mutation,
# so a rejected turn leaves no trace in either.
assert agent.rewind_manager.checkpoints == []
assert len(agent.messages) == initial_message_count
@pytest.mark.asyncio
async def test_act_attaches_images_to_user_message(
png_attachment: ImageAttachment,
) -> None:
config = _config_with_vision_flag(supports_images=True)
backend = FakeBackend([mock_llm_chunk(content="ok")])
agent = build_test_agent_loop(config=config, backend=backend)
[_ async for _ in agent.act("look", images=[png_attachment])]
user_msgs = [m for m in agent.messages if m.role.value == "user"]
assert len(user_msgs) == 1
assert user_msgs[0].images == [png_attachment]
def _seed_history_image(agent, png_attachment: ImageAttachment) -> None:
agent.messages.append(
LLMMessage(role=Role.user, content="earlier", images=[png_attachment])
)
agent.messages.append(LLMMessage(role=Role.assistant, content="seen"))
@pytest.mark.asyncio
@pytest.mark.parametrize("enable_streaming", [False, True])
async def test_history_images_stripped_from_backend_payload_on_non_vision_model(
png_attachment: ImageAttachment, enable_streaming: bool
) -> None:
config = _config_with_both_models()
backend = FakeBackend([mock_llm_chunk(content="ok")])
agent = build_test_agent_loop(
config=config, backend=backend, enable_streaming=enable_streaming
)
_seed_history_image(agent, png_attachment)
agent.config.active_model = "text-alias"
[_ async for _ in agent.act("hi")]
sent_messages = backend.requests_messages[-1]
assert all(m.images is None for m in sent_messages)
# source-of-truth history is not mutated
assert any(m.images == [png_attachment] for m in agent.messages)
@pytest.mark.asyncio
@pytest.mark.parametrize("enable_streaming", [False, True])
async def test_switch_back_to_vision_model_restores_images_in_payload(
png_attachment: ImageAttachment, enable_streaming: bool
) -> None:
config = _config_with_both_models()
backend = FakeBackend([
[mock_llm_chunk(content="a")],
[mock_llm_chunk(content="b")],
])
agent = build_test_agent_loop(
config=config, backend=backend, enable_streaming=enable_streaming
)
_seed_history_image(agent, png_attachment)
agent.config.active_model = "text-alias"
[_ async for _ in agent.act("hi")]
agent.config.active_model = "vision-alias"
[_ async for _ in agent.act("hi again")]
sent_messages = backend.requests_messages[-1]
assert any(m.images == [png_attachment] for m in sent_messages)
def test_count_history_images_unsupported_by_active_model(
png_attachment: ImageAttachment,
) -> None:
config = _config_with_both_models()
agent = build_test_agent_loop(config=config)
assert agent.count_history_images_unsupported_by_active_model() == 0
_seed_history_image(agent, png_attachment)
# active model still supports images
assert agent.count_history_images_unsupported_by_active_model() == 0
agent.config.active_model = "text-alias"
assert agent.count_history_images_unsupported_by_active_model() == 1
agent.messages.append(
LLMMessage(role=Role.user, content="more", images=[png_attachment])
)
assert agent.count_history_images_unsupported_by_active_model() == 2
@pytest.mark.asyncio
async def test_new_images_with_non_vision_model_still_raises(
png_attachment: ImageAttachment,
) -> None:
config = _config_with_both_models()
backend = FakeBackend([mock_llm_chunk(content="ok")])
agent = build_test_agent_loop(config=config, backend=backend)
agent.config.active_model = "text-alias"
with pytest.raises(ImagesNotSupportedError):
async for _ in agent.act("look", images=[png_attachment]):
pass

View file

@ -0,0 +1,33 @@
from __future__ import annotations
import pytest
from tests.conftest import build_test_agent_loop, build_test_vibe_config
@pytest.mark.asyncio
async def test_refresh_system_prompt_preserves_scratchpad_section() -> None:
# Regression: refresh_system_prompt must pass scratchpad_dir, otherwise
# it silently drops the scratchpad instructions from the system prompt.
# This fires on every session start/resume via initialize_experiments
# and hydrate_experiments_from_session, so the LLM would lose awareness
# of the scratchpad on the very first turn for any user with telemetry
# enabled and a Mistral API key.
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=True,
include_model_info=False,
include_commit_signature=False,
)
agent = build_test_agent_loop(config=config)
initial_prompt = agent.messages[0].content or ""
assert "Scratchpad Directory" in initial_prompt
assert agent.scratchpad_dir is not None
assert str(agent.scratchpad_dir) in initial_prompt
await agent.refresh_system_prompt()
refreshed_prompt = agent.messages[0].content or ""
assert "Scratchpad Directory" in refreshed_prompt
assert str(agent.scratchpad_dir) in refreshed_prompt

View file

@ -0,0 +1,831 @@
from __future__ import annotations
from collections import Counter
from collections.abc import Callable
from http import HTTPStatus
from typing import cast
from unittest.mock import AsyncMock, Mock
import httpx
from pydantic import BaseModel
import pytest
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.config import VibeConfig
from vibe.core.llm.exceptions import BackendError, BackendErrorBuilder
from vibe.core.middleware import (
ConversationContext,
MiddlewareAction,
MiddlewareResult,
ResetReason,
)
from vibe.core.tools.builtins.todo import TodoArgs
from vibe.core.types import (
ApprovalResponse,
AssistantEvent,
ContextTooLongError,
FunctionCall,
LLMMessage,
RateLimitError,
ReasoningEvent,
Role,
ToolCall,
ToolCallEvent,
ToolResultEvent,
UserMessageEvent,
)
from vibe.core.utils import CancellationReason, get_user_cancellation_message
class InjectBeforeMiddleware:
injected_message = "<injected>"
async def before_turn(self, context: ConversationContext) -> MiddlewareResult:
"Inject a message just before the current step executes."
return MiddlewareResult(
action=MiddlewareAction.INJECT_MESSAGE, message=self.injected_message
)
def reset(self, reset_reason: ResetReason = ResetReason.STOP) -> None:
return None
def make_config(
*, enabled_tools: list[str] | None = None, tools: dict[str, dict] | None = None
) -> VibeConfig:
return build_test_vibe_config(
system_prompt_id="tests",
include_project_context=False,
include_prompt_detail=False,
include_model_info=False,
include_commit_signature=False,
enabled_tools=enabled_tools or [],
tools=tools or {},
)
@pytest.fixture
def observer_capture() -> tuple[
list[tuple[Role, str | None]], Callable[[LLMMessage], None]
]:
observed: list[tuple[Role, str | None]] = []
def observer(msg: LLMMessage) -> None:
observed.append((msg.role, msg.content))
return observed, observer
@pytest.mark.asyncio
async def test_act_flushes_batched_messages_with_injection_middleware(
observer_capture,
) -> None:
observed, observer = observer_capture
backend = FakeBackend([mock_llm_chunk(content="I can write very efficient code.")])
agent = build_test_agent_loop(
config=make_config(), message_observer=observer, backend=backend
)
agent.middleware_pipeline.add(InjectBeforeMiddleware())
async for _ in agent.act("How can you help?"):
pass
assert len(observed) == 4
assert [r for r, _ in observed] == [
Role.system,
Role.user,
Role.user,
Role.assistant,
]
assert observed[0][1] == "You are Vibe, a super useful programming assistant."
assert observed[1][1] == "How can you help?"
assert observed[2][1] == InjectBeforeMiddleware.injected_message
assert observed[3][1] == "I can write very efficient code."
@pytest.mark.asyncio
async def test_stop_action_flushes_user_msg_before_returning(observer_capture) -> None:
observed, observer = observer_capture
# max_turns=0 forces an immediate STOP on the first before_turn
backend = FakeBackend([
mock_llm_chunk(content="My response will never reach you...")
])
agent = build_test_agent_loop(
config=make_config(), message_observer=observer, max_turns=0, backend=backend
)
async for _ in agent.act("Greet."):
pass
assert len(observed) == 2
# user's message should have been flushed before returning
assert [r for r, _ in observed] == [Role.system, Role.user]
assert observed[0][1] == "You are Vibe, a super useful programming assistant."
assert observed[1][1] == "Greet."
@pytest.mark.asyncio
async def test_act_emits_user_and_assistant_msgs(observer_capture) -> None:
observed, observer = observer_capture
backend = FakeBackend([mock_llm_chunk(content="Pong!")])
agent = build_test_agent_loop(
config=make_config(), message_observer=observer, backend=backend
)
async for _ in agent.act("Ping?"):
pass
assert len(observed) == 3
assert [r for r, _ in observed] == [Role.system, Role.user, Role.assistant]
assert observed[1][1] == "Ping?"
assert observed[2][1] == "Pong!"
@pytest.mark.asyncio
async def test_act_streams_chunks_in_order() -> None:
backend = FakeBackend([
mock_llm_chunk(content="Hello"),
mock_llm_chunk(content=" from"),
mock_llm_chunk(content=" Vibe"),
mock_llm_chunk(content="! "),
mock_llm_chunk(content="More"),
mock_llm_chunk(content=" and"),
mock_llm_chunk(content=" end"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Stream, please.")]
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
assert len(assistant_events) == 7
assert [event.content for event in assistant_events] == [
"Hello",
" from",
" Vibe",
"! ",
"More",
" and",
" end",
]
assert agent.messages[-1].role == Role.assistant
assert agent.messages[-1].content == "Hello from Vibe! More and end"
@pytest.mark.asyncio
async def test_act_streaming_does_not_cleanup_tmp_files_directly() -> None:
backend = FakeBackend([
mock_llm_chunk(content="Hello"),
mock_llm_chunk(content=" from"),
mock_llm_chunk(content=" Vibe"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
cleanup_spy = Mock()
agent.session_logger.maybe_cleanup_tmp_files = cleanup_spy
events = [event async for event in agent.act("Stream, please.")]
assistant_events = [event for event in events if isinstance(event, AssistantEvent)]
assert len(assistant_events) == 3
assert cleanup_spy.call_count == 0
@pytest.mark.asyncio
async def test_act_handles_streaming_with_tool_call_events_in_sequence() -> None:
todo_tool_call = ToolCall(
id="tc_stream",
index=0,
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
)
backend = FakeBackend([
[
mock_llm_chunk(content="Checking your todos."),
mock_llm_chunk(content="", tool_calls=[todo_tool_call]),
],
[mock_llm_chunk(content="Done reviewing todos.")],
])
agent = build_test_agent_loop(
config=make_config(
enabled_tools=["todo"], tools={"todo": {"permission": "always"}}
),
backend=backend,
agent_name=BuiltinAgentName.AUTO_APPROVE,
enable_streaming=True,
)
events = [event async for event in agent.act("What about my todos?")]
assert [type(event) for event in events] == [
UserMessageEvent,
AssistantEvent,
ToolCallEvent,
ToolCallEvent,
ToolResultEvent,
AssistantEvent,
]
assert isinstance(events[0], UserMessageEvent)
assert isinstance(events[1], AssistantEvent)
assert events[1].content == "Checking your todos."
assert isinstance(events[2], ToolCallEvent)
assert events[2].args is None # streaming event
assert events[2].tool_call_id == "tc_stream"
assert events[2].tool_name == "todo"
assert isinstance(events[3], ToolCallEvent)
assert events[3].args is not None
assert events[3].tool_name == "todo"
assert isinstance(events[4], ToolResultEvent)
assert events[4].error is None
assert events[4].skipped is False
assert isinstance(events[5], AssistantEvent)
assert events[5].content == "Done reviewing todos."
assert agent.messages[-1].content == "Done reviewing todos."
@pytest.mark.asyncio
async def test_act_handles_tool_call_chunk_with_content() -> None:
todo_tool_call = ToolCall(
id="tc_content",
index=0,
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
)
backend = FakeBackend([
mock_llm_chunk(content="Preparing "),
mock_llm_chunk(content="todo request", tool_calls=[todo_tool_call]),
mock_llm_chunk(content=" complete"),
])
agent = build_test_agent_loop(
config=make_config(
enabled_tools=["todo"], tools={"todo": {"permission": "always"}}
),
backend=backend,
agent_name=BuiltinAgentName.AUTO_APPROVE,
enable_streaming=True,
)
events = [event async for event in agent.act("Check todos with content.")]
event_types = [type(e) for e in events]
assert Counter(event_types) == Counter({
UserMessageEvent: 1,
AssistantEvent: 3,
ToolCallEvent: 2,
ToolResultEvent: 1,
})
tool_call_events = [e for e in events if isinstance(e, ToolCallEvent)]
assert len(tool_call_events) == 2
assert any(
tc.args is None and tc.tool_call_id == "tc_content" for tc in tool_call_events
)
assert any(tc.args is not None for tc in tool_call_events)
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
assistant_contents = {e.content for e in assistant_events}
assert "Preparing " in assistant_contents
assert "todo request" in assistant_contents
assert " complete" in assistant_contents
assert any(
m.role == Role.assistant and m.content == "Preparing todo request complete"
for m in agent.messages
)
@pytest.mark.asyncio
async def test_act_merges_streamed_tool_call_arguments() -> None:
tool_call_part_one = ToolCall(
id="tc_merge",
index=0,
function=FunctionCall(
name="todo", arguments='{"action": "read", "note": "First '
),
)
tool_call_part_two = ToolCall(
id="tc_merge", index=0, function=FunctionCall(name="todo", arguments='part"}')
)
backend = FakeBackend([
mock_llm_chunk(content="Planning: "),
mock_llm_chunk(content="", tool_calls=[tool_call_part_one]),
mock_llm_chunk(content="", tool_calls=[tool_call_part_two]),
])
agent = build_test_agent_loop(
config=make_config(
enabled_tools=["todo"], tools={"todo": {"permission": "always"}}
),
backend=backend,
agent_name=BuiltinAgentName.AUTO_APPROVE,
enable_streaming=True,
)
events = [event async for event in agent.act("Merge streamed tool call args.")]
assert [type(event) for event in events] == [
UserMessageEvent,
AssistantEvent,
ToolCallEvent,
ToolCallEvent,
ToolResultEvent,
]
assert isinstance(events[0], UserMessageEvent)
assert isinstance(events[2], ToolCallEvent)
assert events[2].args is None # streaming event
assert events[2].tool_call_id == "tc_merge"
call_event = events[3]
assert isinstance(call_event, ToolCallEvent)
assert call_event.tool_call_id == "tc_merge"
call_args = cast(TodoArgs, call_event.args)
assert call_args.action == "read"
assert isinstance(events[4], ToolResultEvent)
assert events[4].error is None
assert events[4].skipped is False
assistant_with_calls = next(
m for m in agent.messages if m.role == Role.assistant and m.tool_calls
)
reconstructed_calls = assistant_with_calls.tool_calls or []
assert len(reconstructed_calls) == 1
assert reconstructed_calls[0].function.arguments == (
'{"action": "read", "note": "First part"}'
)
@pytest.mark.asyncio
async def test_act_handles_user_cancellation_during_streaming() -> None:
class CountingMiddleware:
def __init__(self) -> None:
self.before_calls = 0
async def before_turn(self, context: ConversationContext) -> MiddlewareResult:
self.before_calls += 1
return MiddlewareResult()
def reset(self, reset_reason: ResetReason = ResetReason.STOP) -> None:
return None
todo_tool_call = ToolCall(
id="tc_cancel",
index=0,
function=FunctionCall(name="todo", arguments='{"action": "read"}'),
)
backend = FakeBackend([
mock_llm_chunk(content="Preparing "),
mock_llm_chunk(content="todo request", tool_calls=[todo_tool_call]),
])
agent = build_test_agent_loop(
config=make_config(
enabled_tools=["todo"], tools={"todo": {"permission": "ask"}}
),
backend=backend,
agent_name=BuiltinAgentName.DEFAULT,
enable_streaming=True,
)
middleware = CountingMiddleware()
agent.middleware_pipeline.add(middleware)
async def _reject_callback(
_name: str, _args: BaseModel, _id: str, _rp: list | None = None
) -> tuple[ApprovalResponse, str | None]:
return (
ApprovalResponse.NO,
str(get_user_cancellation_message(CancellationReason.OPERATION_CANCELLED)),
)
agent.set_approval_callback(_reject_callback)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
events = [event async for event in agent.act("Cancel mid stream?")]
assert [type(event) for event in events] == [
UserMessageEvent,
AssistantEvent,
ToolCallEvent,
AssistantEvent,
ToolCallEvent,
ToolResultEvent,
]
assert middleware.before_calls == 1
assert isinstance(events[-1], ToolResultEvent)
assert events[-1].skipped is True
assert events[-1].skip_reason is not None
assert "<user_cancellation>" in events[-1].skip_reason
assert events[-1].cancelled is True
assert agent.session_logger.save_interaction.await_count >= 1
@pytest.mark.asyncio
async def test_act_flushes_and_logs_when_streaming_errors(observer_capture) -> None:
observed, observer = observer_capture
backend = FakeBackend(exception_to_raise=RuntimeError("boom in streaming"))
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=True,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(RuntimeError, match="boom in streaming"):
[_ async for _ in agent.act("Trigger stream failure")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
@pytest.mark.asyncio
async def test_rate_limit(observer_capture) -> None:
observed, observer = observer_capture
response = httpx.Response(
HTTPStatus.TOO_MANY_REQUESTS, request=httpx.Request("POST", "http://test")
)
error = httpx.HTTPStatusError(
"rate limited", request=response.request, response=response
)
backend_error = BackendErrorBuilder.build_http_error(
provider="mistral",
endpoint="test",
error=error,
model="test-model",
messages=[],
temperature=0.0,
has_tools=False,
tool_choice=None,
)
backend = FakeBackend(exception_to_raise=backend_error)
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=True,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(RateLimitError):
[_ async for _ in agent.act("Trigger rate limit failure while streaming")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
def _build_context_too_long_backend_error() -> BackendError:
response = httpx.Response(
HTTPStatus.BAD_REQUEST,
request=httpx.Request("POST", "http://test"),
text='{"message": "Context too long"}',
)
error = httpx.HTTPStatusError(
"context too long", request=response.request, response=response
)
return BackendErrorBuilder.build_http_error(
provider="mistral",
endpoint="test",
error=error,
model="test-model",
messages=[],
temperature=0.0,
has_tools=False,
tool_choice=None,
)
@pytest.mark.asyncio
async def test_context_too_long_streaming(observer_capture) -> None:
observed, observer = observer_capture
backend_error = _build_context_too_long_backend_error()
backend = FakeBackend(exception_to_raise=backend_error)
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=True,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(ContextTooLongError):
[_ async for _ in agent.act("Trigger context too long while streaming")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
@pytest.mark.asyncio
async def test_context_too_long_non_streaming(observer_capture) -> None:
observed, observer = observer_capture
backend_error = _build_context_too_long_backend_error()
backend = FakeBackend(exception_to_raise=backend_error)
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=False,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(ContextTooLongError):
[_ async for _ in agent.act("Trigger context too long without streaming")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
class _NonRetryableError(Exception):
# Mimics Temporal's ``ApplicationError(non_retryable=True)`` without
# pulling temporalio into vibe's test deps. The wrap-site check relies
# on the truthy ``non_retryable`` attribute, not the concrete type.
non_retryable = True
@pytest.mark.asyncio
async def test_non_retryable_passes_through_streaming(observer_capture) -> None:
observed, observer = observer_capture
backend = FakeBackend(exception_to_raise=_NonRetryableError("auth failed"))
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=True,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(_NonRetryableError, match="auth failed"):
[_ async for _ in agent.act("Trigger non-retryable failure while streaming")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
@pytest.mark.asyncio
async def test_non_retryable_passes_through_non_streaming(observer_capture) -> None:
observed, observer = observer_capture
backend = FakeBackend(exception_to_raise=_NonRetryableError("auth failed"))
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=False,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(_NonRetryableError, match="auth failed"):
[_ async for _ in agent.act("Trigger non-retryable failure without streaming")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
def _wrap_with_cause(message: str, cause: BaseException) -> RuntimeError:
# Mirrors how Temporal raises ``ActivityError`` on the workflow side with
# the original ``ApplicationError`` chained as ``__cause__`` — except we
# don't import temporalio. The wrap-site check has to traverse the chain
# to find ``non_retryable`` regardless of how deep it is.
error = RuntimeError(message)
error.__cause__ = cause
return error
@pytest.mark.asyncio
async def test_non_retryable_via_cause_chain_streaming(observer_capture) -> None:
observed, observer = observer_capture
wrapped = _wrap_with_cause(
"Activity task failed", _NonRetryableError("auth failed")
)
backend = FakeBackend(exception_to_raise=wrapped)
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=True,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(RuntimeError, match="Activity task failed"):
[_ async for _ in agent.act("Trigger non-retryable via cause chain")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
@pytest.mark.asyncio
async def test_non_retryable_via_cause_chain_non_streaming(observer_capture) -> None:
observed, observer = observer_capture
wrapped = _wrap_with_cause(
"Activity task failed", _NonRetryableError("auth failed")
)
backend = FakeBackend(exception_to_raise=wrapped)
agent = build_test_agent_loop(
config=make_config(),
backend=backend,
message_observer=observer,
enable_streaming=False,
)
agent.session_logger.save_interaction = AsyncMock(return_value=None)
with pytest.raises(RuntimeError, match="Activity task failed"):
[_ async for _ in agent.act("Trigger non-retryable via cause chain")]
assert [role for role, _ in observed] == [Role.system, Role.user]
assert agent.session_logger.save_interaction.await_count == 1
def _snapshot_events(events: list) -> list[tuple[str, str]]:
return [
(type(e).__name__, e.content)
for e in events
if isinstance(e, (AssistantEvent, ReasoningEvent))
]
@pytest.mark.asyncio
async def test_reasoning_yields_before_content_on_transition() -> None:
backend = FakeBackend([
mock_llm_chunk(content="", reasoning_content="Let me think"),
mock_llm_chunk(content="", reasoning_content=" about this"),
mock_llm_chunk(content="", reasoning_content=" problem..."),
mock_llm_chunk(content="The answer is 42."),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("What's the answer?")]
assert _snapshot_events(events) == [
("ReasoningEvent", "Let me think"),
("ReasoningEvent", " about this"),
("ReasoningEvent", " problem..."),
("AssistantEvent", "The answer is 42."),
]
@pytest.mark.asyncio
async def test_reasoning_yields_per_chunk() -> None:
backend = FakeBackend([
mock_llm_chunk(content="", reasoning_content="Step 1"),
mock_llm_chunk(content="", reasoning_content=", Step 2"),
mock_llm_chunk(content="", reasoning_content=", Step 3"),
mock_llm_chunk(content="", reasoning_content=", Step 4"),
mock_llm_chunk(content="", reasoning_content=", Step 5"),
mock_llm_chunk(content="", reasoning_content=", Step 6"),
mock_llm_chunk(content="", reasoning_content=", Final"),
mock_llm_chunk(content="Done thinking!"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Think step by step")]
assert _snapshot_events(events) == [
("ReasoningEvent", "Step 1"),
("ReasoningEvent", ", Step 2"),
("ReasoningEvent", ", Step 3"),
("ReasoningEvent", ", Step 4"),
("ReasoningEvent", ", Step 5"),
("ReasoningEvent", ", Step 6"),
("ReasoningEvent", ", Final"),
("AssistantEvent", "Done thinking!"),
]
@pytest.mark.asyncio
async def test_content_yields_before_reasoning_on_transition() -> None:
"""When content chunks arrive and reasoning arrives, content yields first."""
backend = FakeBackend([
mock_llm_chunk(content="Starting the response"),
mock_llm_chunk(content=" here..."),
mock_llm_chunk(content="", reasoning_content="Wait, let me reconsider"),
mock_llm_chunk(content="", reasoning_content=" this approach..."),
mock_llm_chunk(content="Actually, the final answer."),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Give me an answer")]
assert _snapshot_events(events) == [
("AssistantEvent", "Starting the response"),
("AssistantEvent", " here..."),
("ReasoningEvent", "Wait, let me reconsider"),
("ReasoningEvent", " this approach..."),
("AssistantEvent", "Actually, the final answer."),
]
@pytest.mark.asyncio
async def test_interleaved_reasoning_content_preserves_order() -> None:
backend = FakeBackend([
mock_llm_chunk(content="", reasoning_content="Think 1"),
mock_llm_chunk(content="Answer 1 "),
mock_llm_chunk(content="", reasoning_content="Think 2"),
mock_llm_chunk(content="Answer 2 "),
mock_llm_chunk(content="", reasoning_content="Think 3"),
mock_llm_chunk(content="Answer 3"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Interleaved test")]
assert _snapshot_events(events) == [
("ReasoningEvent", "Think 1"),
("AssistantEvent", "Answer 1 "),
("ReasoningEvent", "Think 2"),
("AssistantEvent", "Answer 2 "),
("ReasoningEvent", "Think 3"),
("AssistantEvent", "Answer 3"),
]
assistant_msg = next(m for m in agent.messages if m.role == Role.assistant)
assert assistant_msg.reasoning_content == "Think 1Think 2Think 3"
assert assistant_msg.content == "Answer 1 Answer 2 Answer 3"
@pytest.mark.asyncio
async def test_only_reasoning_chunks_yields_reasoning_event() -> None:
backend = FakeBackend([
mock_llm_chunk(content="", reasoning_content="Just thinking..."),
mock_llm_chunk(content="", reasoning_content=" nothing to say yet."),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Silent thinking")]
assert _snapshot_events(events) == [
("ReasoningEvent", "Just thinking..."),
("ReasoningEvent", " nothing to say yet."),
]
@pytest.mark.asyncio
async def test_final_buffers_flush_in_correct_order() -> None:
backend = FakeBackend([
mock_llm_chunk(content="", reasoning_content="Final thought"),
mock_llm_chunk(content="Final words"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("End buffers test")]
assert _snapshot_events(events) == [
("ReasoningEvent", "Final thought"),
("AssistantEvent", "Final words"),
]
@pytest.mark.asyncio
async def test_empty_content_chunks_do_not_trigger_false_yields() -> None:
backend = FakeBackend([
mock_llm_chunk(content="", reasoning_content="Reasoning here"),
mock_llm_chunk(content=""), # Empty content shouldn't yield
mock_llm_chunk(content="", reasoning_content=" more reasoning"),
mock_llm_chunk(content="Actual content"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Empty content test")]
assert _snapshot_events(events) == [
("ReasoningEvent", "Reasoning here"),
("ReasoningEvent", " more reasoning"),
("AssistantEvent", "Actual content"),
]
@pytest.mark.asyncio
async def test_streaming_assistant_event_message_id_matches_stored_message() -> None:
backend = FakeBackend([
mock_llm_chunk(content="Hello"),
mock_llm_chunk(content=" world"),
])
agent = build_test_agent_loop(
config=make_config(), backend=backend, enable_streaming=True
)
events = [event async for event in agent.act("Test")]
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
assert len(assistant_events) == 2
# All chunks of the same assistant turn share one message_id
message_ids = {e.message_id for e in assistant_events}
assert len(message_ids) == 1
# The stored LLMMessage must carry that same message_id
stored_msg = next(m for m in agent.messages if m.role == Role.assistant)
assert stored_msg.message_id == assistant_events[0].message_id

View file

@ -0,0 +1,109 @@
from __future__ import annotations
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.paths import PLANS_DIR
from vibe.core.tools.base import ToolPermission
class TestPlanAgentWriteFileResolvePermission:
"""Plan agent sets write_file to NEVER with allowlist=[plans/*].
resolve_permission must use this, not the base config.
"""
def test_write_file_to_non_plan_path_denied_in_plan_mode(self) -> None:
config = build_test_vibe_config()
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
tool = agent.tool_manager.get("write_file")
from vibe.core.tools.builtins.write_file import WriteFileArgs
args = WriteFileArgs(path="/some/random/file.py", content="hello")
ctx = tool.resolve_permission(args)
# With plan agent override: permission should be NEVER
# (unless the path matches the plans allowlist)
assert ctx is not None
assert ctx.permission == ToolPermission.NEVER
def test_write_file_to_plan_path_allowed_in_plan_mode(self) -> None:
config = build_test_vibe_config()
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
tool = agent.tool_manager.get("write_file")
from vibe.core.tools.builtins.write_file import WriteFileArgs
plan_path = str(PLANS_DIR.path / "my-plan.md")
args = WriteFileArgs(path=plan_path, content="# Plan")
ctx = tool.resolve_permission(args)
# Plan path is in the allowlist, so should be ALWAYS
assert ctx is not None
assert ctx.permission == ToolPermission.ALWAYS
def test_edit_to_non_plan_path_denied_in_plan_mode(self) -> None:
config = build_test_vibe_config()
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
tool = agent.tool_manager.get("edit")
from vibe.core.tools.builtins.edit import EditArgs
args = EditArgs(file_path="/some/file.py", old_string="a", new_string="b")
ctx = tool.resolve_permission(args)
assert ctx is not None
assert ctx.permission == ToolPermission.NEVER
class TestAcceptEditsAgentResolvePermission:
"""Accept-edits agent sets write_file/edit to ALWAYS.
resolve_permission must reflect this.
"""
def test_write_file_always_in_accept_edits_mode(self) -> None:
config = build_test_vibe_config()
agent = build_test_agent_loop(
config=config, agent_name=BuiltinAgentName.ACCEPT_EDITS
)
tool = agent.tool_manager.get("write_file")
from vibe.core.tools.builtins.write_file import WriteFileArgs
# Use a workdir-relative path; outside-workdir always requires ASK
# regardless of agent permission.
args = WriteFileArgs(path="file.py", content="hello")
ctx = tool.resolve_permission(args)
# Inside workdir, no allowlist/denylist/sensitive match → None,
# so the caller falls through to config permission (ALWAYS).
assert ctx is None
class TestAgentOverrideNotLeakedAcrossSwitches:
"""Switching agents must change what resolve_permission returns."""
def test_switch_from_plan_to_default_restores_write_permission(self) -> None:
config = build_test_vibe_config()
agent = build_test_agent_loop(config=config, agent_name=BuiltinAgentName.PLAN)
tool = agent.tool_manager.get("write_file")
from vibe.core.tools.builtins.write_file import WriteFileArgs
args = WriteFileArgs(path="/some/file.py", content="hello")
# In plan mode: should be NEVER
ctx_plan = tool.resolve_permission(args)
assert ctx_plan is not None
assert ctx_plan.permission == ToolPermission.NEVER
# Switch to default
agent.agent_manager.switch_profile(BuiltinAgentName.DEFAULT)
# In default mode: should NOT be NEVER
ctx_default = tool.resolve_permission(args)
assert ctx_default is not None
assert ctx_default.permission != ToolPermission.NEVER

View file

@ -0,0 +1,752 @@
from __future__ import annotations
from collections.abc import Callable
import pytest
from tests.conftest import (
build_test_agent_loop,
build_test_vibe_config,
make_test_models,
)
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.config import (
ModelConfig,
ProviderConfig,
SessionLoggingConfig,
VibeConfig,
)
from vibe.core.tools.base import ToolPermission
from vibe.core.types import (
AgentStats,
AssistantEvent,
Backend,
CompactEndEvent,
CompactStartEvent,
FunctionCall,
LLMMessage,
Role,
ToolCall,
UserMessageEvent,
)
def make_config(
*,
system_prompt_id: str = "tests",
active_model: str = "devstral-latest",
input_price: float = 0.4,
output_price: float = 2.0,
disable_logging: bool = True,
auto_compact_threshold: int = 0,
include_project_context: bool = False,
include_prompt_detail: bool = False,
enabled_tools: list[str] | None = None,
todo_permission: ToolPermission = ToolPermission.ALWAYS,
) -> VibeConfig:
models = [
ModelConfig(
name="mistral-vibe-cli-latest",
provider="mistral",
alias="devstral-latest",
input_price=input_price,
output_price=output_price,
auto_compact_threshold=auto_compact_threshold,
),
ModelConfig(
name="devstral-small-latest",
provider="mistral",
alias="devstral-small",
input_price=0.1,
output_price=0.3,
auto_compact_threshold=auto_compact_threshold,
),
ModelConfig(
name="strawberry",
provider="lechat",
alias="strawberry",
input_price=2.5,
output_price=10.0,
auto_compact_threshold=auto_compact_threshold,
),
]
providers = [
ProviderConfig(
name="mistral",
api_base="https://api.mistral.ai/v1",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
),
ProviderConfig(
name="lechat",
api_base="https://api.mistral.ai/v1",
api_key_env_var="LECHAT_API_KEY",
backend=Backend.MISTRAL,
),
]
return build_test_vibe_config(
session_logging=SessionLoggingConfig(enabled=not disable_logging),
system_prompt_id=system_prompt_id,
include_project_context=include_project_context,
include_prompt_detail=include_prompt_detail,
active_model=active_model,
models=models,
providers=providers,
enabled_tools=enabled_tools or [],
tools={"todo": {"permission": todo_permission.value}},
)
@pytest.fixture
def observer_capture() -> tuple[list[LLMMessage], Callable[[LLMMessage], None]]:
observed: list[LLMMessage] = []
def observer(msg: LLMMessage) -> None:
observed.append(msg)
return observed, observer
class TestAgentStatsHelpers:
def test_update_pricing(self) -> None:
stats = AgentStats()
stats.update_pricing(1.5, 3.0)
assert stats.input_price_per_million == 1.5
assert stats.output_price_per_million == 3.0
def test_reset_context_state_preserves_cumulative(self) -> None:
stats = AgentStats(
steps=5,
session_prompt_tokens=1000,
session_completion_tokens=500,
tool_calls_succeeded=3,
tool_calls_failed=1,
context_tokens=800,
last_turn_prompt_tokens=100,
last_turn_completion_tokens=50,
last_turn_duration=1.5,
tokens_per_second=33.3,
input_price_per_million=0.4,
output_price_per_million=2.0,
)
stats.reset_context_state()
assert stats.steps == 5
assert stats.session_prompt_tokens == 1000
assert stats.session_completion_tokens == 500
assert stats.tool_calls_succeeded == 3
assert stats.tool_calls_failed == 1
assert stats.input_price_per_million == 0.4
assert stats.output_price_per_million == 2.0
assert stats.context_tokens == 0
assert stats.last_turn_prompt_tokens == 0
assert stats.last_turn_completion_tokens == 0
assert stats.last_turn_duration == 0.0
assert stats.tokens_per_second == 0.0
def test_session_cost_computed_from_current_pricing(self) -> None:
stats = AgentStats(
session_prompt_tokens=1_000_000,
session_completion_tokens=500_000,
input_price_per_million=1.0,
output_price_per_million=2.0,
)
# Cost = 1M * $1/M + 0.5M * $2/M = $1 + $1 = $2
assert stats.session_cost == 2.0
stats.update_pricing(2.0, 4.0)
# Cost = 1M * $2/M + 0.5M * $4/M = $2 + $2 = $4
assert stats.session_cost == 4.0
class TestReloadPreservesStats:
@pytest.mark.asyncio
async def test_reload_preserves_session_tokens(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="First response"))
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Hello"):
pass
old_session_prompt = agent.stats.session_prompt_tokens
old_session_completion = agent.stats.session_completion_tokens
assert old_session_prompt > 0
assert old_session_completion > 0
await agent.reload_with_initial_messages()
assert agent.stats.session_prompt_tokens == old_session_prompt
assert agent.stats.session_completion_tokens == old_session_completion
@pytest.mark.asyncio
async def test_reload_preserves_tool_call_stats(self) -> None:
backend = FakeBackend([
mock_llm_chunk(
content="Calling tool",
tool_calls=[
ToolCall(
id="tc1",
index=0,
function=FunctionCall(
name="todo", arguments='{"action": "read"}'
),
)
],
),
mock_llm_chunk(content="Done"),
])
config = make_config(enabled_tools=["todo"])
agent = build_test_agent_loop(
config=config, agent_name=BuiltinAgentName.AUTO_APPROVE, backend=backend
)
async for _ in agent.act("Check todos"):
pass
assert agent.stats.tool_calls_succeeded == 1
assert agent.stats.tool_calls_agreed == 1
await agent.reload_with_initial_messages()
assert agent.stats.tool_calls_succeeded == 1
assert agent.stats.tool_calls_agreed == 1
@pytest.mark.asyncio
async def test_reload_preserves_steps(self) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="R1")],
[mock_llm_chunk(content="R2")],
])
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("First"):
pass
async for _ in agent.act("Second"):
pass
old_steps = agent.stats.steps
assert old_steps >= 2
await agent.reload_with_initial_messages()
assert agent.stats.steps == old_steps
@pytest.mark.asyncio
async def test_reload_preserves_context_tokens_when_messages_preserved(
self,
) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(config=make_config(), backend=backend)
[_ async for _ in agent.act("Hello")]
assert agent.stats.context_tokens > 0
initial_context_tokens = agent.stats.context_tokens
assert len(agent.messages) > 1
await agent.reload_with_initial_messages()
assert len(agent.messages) > 1
assert agent.stats.context_tokens == initial_context_tokens
@pytest.mark.asyncio
async def test_reload_resets_context_tokens_when_no_messages(self) -> None:
backend = FakeBackend([])
agent = build_test_agent_loop(config=make_config(), backend=backend)
assert len(agent.messages) == 1
assert agent.stats.context_tokens == 0
await agent.reload_with_initial_messages()
assert len(agent.messages) == 1
assert agent.stats.context_tokens == 0
@pytest.mark.asyncio
async def test_reload_resets_context_tokens_when_system_prompt_changes(
self,
) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
config1 = make_config(system_prompt_id="tests")
config2 = make_config(system_prompt_id="cli")
agent = build_test_agent_loop(config=config1, backend=backend)
[_ async for _ in agent.act("Hello")]
original_context_tokens = agent.stats.context_tokens
assert original_context_tokens > 0
assert len(agent.messages) > 1
await agent.reload_with_initial_messages(base_config=config2)
assert len(agent.messages) > 1
assert agent.stats.context_tokens == original_context_tokens
@pytest.mark.asyncio
async def test_reload_updates_pricing_from_new_model(self, monkeypatch) -> None:
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
backend = FakeBackend(mock_llm_chunk(content="Response"))
config_mistral = make_config(active_model="devstral-latest")
agent = build_test_agent_loop(config=config_mistral, backend=backend)
async for _ in agent.act("Hello"):
pass
assert agent.stats.input_price_per_million == 0.4
assert agent.stats.output_price_per_million == 2.0
config_other = make_config(active_model="strawberry")
await agent.reload_with_initial_messages(base_config=config_other)
assert agent.stats.input_price_per_million == 2.5
assert agent.stats.output_price_per_million == 10.0
@pytest.mark.asyncio
async def test_reload_accumulates_tokens_across_configs(self, monkeypatch) -> None:
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
backend = FakeBackend([
[mock_llm_chunk(content="First")],
[mock_llm_chunk(content="After reload")],
])
config1 = make_config(active_model="devstral-latest")
agent = build_test_agent_loop(config=config1, backend=backend)
async for _ in agent.act("Hello"):
pass
tokens_after_first = (
agent.stats.session_prompt_tokens + agent.stats.session_completion_tokens
)
config2 = make_config(active_model="strawberry")
await agent.reload_with_initial_messages(base_config=config2)
async for _ in agent.act("Continue"):
pass
tokens_after_second = (
agent.stats.session_prompt_tokens + agent.stats.session_completion_tokens
)
assert tokens_after_second > tokens_after_first
class TestReloadPreservesMessages:
@pytest.mark.asyncio
async def test_reload_preserves_conversation_messages(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Hello"):
pass
assert len(agent.messages) == 3
old_user_content = agent.messages[1].content
old_assistant_content = agent.messages[2].content
await agent.reload_with_initial_messages()
assert len(agent.messages) == 3
assert agent.messages[0].role == Role.system
assert agent.messages[1].role == Role.user
assert agent.messages[1].content == old_user_content
assert agent.messages[2].role == Role.assistant
assert agent.messages[2].content == old_assistant_content
@pytest.mark.asyncio
async def test_reload_updates_system_prompt_preserves_rest(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
config1 = make_config(system_prompt_id="tests")
agent = build_test_agent_loop(config=config1, backend=backend)
async for _ in agent.act("Hello"):
pass
old_system = agent.messages[0].content
old_user = agent.messages[1].content
config2 = make_config(system_prompt_id="cli")
await agent.reload_with_initial_messages(base_config=config2)
assert agent.messages[0].content != old_system
assert agent.messages[1].content == old_user
@pytest.mark.asyncio
async def test_reload_with_no_messages_stays_empty(self) -> None:
backend = FakeBackend([])
agent = build_test_agent_loop(config=make_config(), backend=backend)
assert len(agent.messages) == 1
await agent.reload_with_initial_messages()
assert len(agent.messages) == 1
assert agent.messages[0].role == Role.system
@pytest.mark.asyncio
async def test_reload_does_not_reemit_to_observer(self, observer_capture) -> None:
observed, observer = observer_capture
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(
config=make_config(), message_observer=observer, backend=backend
)
async for _ in agent.act("Hello"):
pass
observed.clear()
await agent.reload_with_initial_messages()
assert len(observed) == 0
class TestCompactStatsHandling:
@pytest.mark.asyncio
async def test_compact_preserves_cumulative_stats(self) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="First response")],
[mock_llm_chunk(content="<summary>")],
])
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Build something"):
pass
tokens_before_compact = agent.stats.session_prompt_tokens
completions_before = agent.stats.session_completion_tokens
steps_before = agent.stats.steps
await agent.compact()
# Cumulative stats include the compact turn
assert agent.stats.session_prompt_tokens > tokens_before_compact
assert agent.stats.session_completion_tokens > completions_before
assert agent.stats.steps > steps_before
@pytest.mark.asyncio
async def test_compact_updates_context_tokens(self) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="Long response " * 100)],
[mock_llm_chunk(content="<summary>")],
])
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Do something complex"):
pass
context_before = agent.stats.context_tokens
await agent.compact()
assert agent.stats.context_tokens < context_before
@pytest.mark.asyncio
async def test_compact_preserves_tool_call_stats(self) -> None:
backend = FakeBackend([
[
mock_llm_chunk(
content="Using tool",
tool_calls=[
ToolCall(
id="tc1",
index=0,
function=FunctionCall(
name="todo", arguments='{"action": "read"}'
),
)
],
),
mock_llm_chunk(content=" todo"),
],
[mock_llm_chunk(content="<summary>")],
])
config = make_config(enabled_tools=["todo"])
agent = build_test_agent_loop(
config=config, agent_name=BuiltinAgentName.AUTO_APPROVE, backend=backend
)
async for _ in agent.act("Check todos"):
pass
assert agent.stats.tool_calls_succeeded == 1
await agent.compact()
assert agent.stats.tool_calls_succeeded == 1
@pytest.mark.asyncio
async def test_compact_resets_session_id(self) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="Long response " * 100)],
[mock_llm_chunk(content="<summary>")],
])
agent = build_test_agent_loop(
config=make_config(disable_logging=False), backend=backend
)
original_session_id = agent.session_id
original_logger_session_id = agent.session_logger.session_id
assert agent.session_id == original_logger_session_id
async for _ in agent.act("Do something complex"):
pass
await agent.compact()
assert agent.session_id != original_session_id
assert agent.session_id == agent.session_logger.session_id
class TestAutoCompactIntegration:
@pytest.mark.asyncio
async def test_auto_compact_triggers_and_preserves_stats(self) -> None:
observed: list[tuple[Role, str | None]] = []
def observer(msg: LLMMessage) -> None:
observed.append((msg.role, msg.content))
backend = FakeBackend([
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="<final>")],
])
cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1))
agent = build_test_agent_loop(
config=cfg, message_observer=observer, backend=backend
)
agent.stats.context_tokens = 2
events = [ev async for ev in agent.act("Hello")]
assert len(events) == 4
assert isinstance(events[0], UserMessageEvent)
assert isinstance(events[1], CompactStartEvent)
assert isinstance(events[2], CompactEndEvent)
assert isinstance(events[3], AssistantEvent)
start: CompactStartEvent = events[1]
final: AssistantEvent = events[3]
assert start.current_context_tokens == 2
assert start.threshold == 1
assert final.content == "<final>"
roles = [r for r, _ in observed]
assert roles == [Role.system, Role.user, Role.assistant]
assert observed[1][1] == "Hello"
class TestClearHistoryFullReset:
@pytest.mark.asyncio
async def test_clear_history_preserves_listeners(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(config=make_config(), backend=backend)
listener_calls: list[int] = []
agent.stats.add_listener(
"context_tokens", lambda s: listener_calls.append(s.context_tokens)
)
async for _ in agent.act("Hello"):
pass
assert agent.stats.context_tokens > 0
listener_calls.clear()
await agent.clear_history()
assert agent.stats.context_tokens == 0
assert any(v == 0 for v in listener_calls)
@pytest.mark.asyncio
async def test_clear_history_fully_resets_stats(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Hello"):
pass
assert agent.stats.session_prompt_tokens > 0
assert agent.stats.steps > 0
await agent.clear_history()
assert agent.stats.session_prompt_tokens == 0
assert agent.stats.session_completion_tokens == 0
assert agent.stats.steps == 0
@pytest.mark.asyncio
async def test_clear_history_preserves_pricing(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
config = make_config(input_price=0.4, output_price=2.0)
agent = build_test_agent_loop(config=config, backend=backend)
async for _ in agent.act("Hello"):
pass
await agent.clear_history()
assert agent.stats.input_price_per_million == 0.4
assert agent.stats.output_price_per_million == 2.0
@pytest.mark.asyncio
async def test_clear_history_removes_messages(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Hello"):
pass
assert len(agent.messages) == 3
await agent.clear_history()
assert len(agent.messages) == 1
assert agent.messages[0].role == Role.system
@pytest.mark.asyncio
async def test_clear_history_resets_session_id(self) -> None:
backend = FakeBackend(mock_llm_chunk(content="Response"))
agent = build_test_agent_loop(
config=make_config(disable_logging=False), backend=backend
)
original_session_id = agent.session_id
original_logger_session_id = agent.session_logger.session_id
assert agent.session_id == original_logger_session_id
async for _ in agent.act("Hello"):
pass
await agent.clear_history()
assert agent.session_id != original_session_id
assert agent.session_id == agent.session_logger.session_id
assert agent.parent_session_id is None
class TestClearHistoryObserverBugfix:
@pytest.mark.asyncio
async def test_clear_history_observer_sees_new_messages(
self, observer_capture
) -> None:
"""Bug fix: clear_history previously left a stale index, so new messages
appended after clearing were never observed.
"""
observed, observer = observer_capture
backend = FakeBackend([
[mock_llm_chunk(content="First")],
[mock_llm_chunk(content="Second")],
])
agent = build_test_agent_loop(
config=make_config(), message_observer=observer, backend=backend
)
async for _ in agent.act("Hello"):
pass
await agent.clear_history()
observed.clear()
async for _ in agent.act("After clear"):
pass
roles = [msg.role for msg in observed]
assert Role.user in roles
assert Role.assistant in roles
class TestStatsEdgeCases:
@pytest.mark.asyncio
async def test_session_cost_approximation_on_model_change(
self, monkeypatch
) -> None:
monkeypatch.setenv("LECHAT_API_KEY", "mock-key")
backend = FakeBackend(mock_llm_chunk(content="Response"))
config1 = make_config(active_model="devstral-latest")
agent = build_test_agent_loop(config=config1, backend=backend)
async for _ in agent.act("Hello"):
pass
cost_before = agent.stats.session_cost
config2 = make_config(active_model="strawberry")
await agent.reload_with_initial_messages(base_config=config2)
cost_after = agent.stats.session_cost
assert cost_after > cost_before
@pytest.mark.asyncio
async def test_multiple_reloads_accumulate_correctly(self) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="R1")],
[mock_llm_chunk(content="R2")],
[mock_llm_chunk(content="R3")],
])
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("First"):
pass
tokens1 = agent.stats.session_total_llm_tokens
await agent.reload_with_initial_messages()
async for _ in agent.act("Second"):
pass
tokens2 = agent.stats.session_total_llm_tokens
await agent.reload_with_initial_messages()
async for _ in agent.act("Third"):
pass
tokens3 = agent.stats.session_total_llm_tokens
assert tokens1 < tokens2 < tokens3
@pytest.mark.asyncio
async def test_compact_then_reload_preserves_both(self) -> None:
backend = FakeBackend([
[mock_llm_chunk(content="Initial response")],
[mock_llm_chunk(content="<summary>")],
[mock_llm_chunk(content="After reload")],
])
agent = build_test_agent_loop(config=make_config(), backend=backend)
async for _ in agent.act("Build something"):
pass
await agent.compact()
tokens_after_compact = agent.stats.session_prompt_tokens
await agent.reload_with_initial_messages()
assert agent.stats.session_prompt_tokens == tokens_after_compact
async for _ in agent.act("Continue"):
pass
assert agent.stats.session_prompt_tokens > tokens_after_compact
@pytest.mark.asyncio
async def test_reload_without_config_preserves_current(self) -> None:
backend = FakeBackend([])
original_config = make_config(active_model="devstral-latest")
agent = build_test_agent_loop(config=original_config, backend=backend)
await agent.reload_with_initial_messages(base_config=None)
assert agent.config.active_model == "devstral-latest"
@pytest.mark.asyncio
async def test_reload_with_new_config_updates_it(self) -> None:
backend = FakeBackend([])
original_config = make_config(active_model="devstral-latest")
agent = build_test_agent_loop(config=original_config, backend=backend)
new_config = make_config(active_model="devstral-small")
await agent.reload_with_initial_messages(base_config=new_config)
assert agent.config.active_model == "devstral-small"

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,765 @@
from __future__ import annotations
import ast
from pathlib import Path
import pytest
from tests import TESTS_ROOT
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.stubs.fake_backend import FakeBackend
from vibe.core.agents.manager import AgentManager
from vibe.core.agents.models import (
BUILTIN_AGENTS,
CHAT,
AgentProfile,
AgentSafety,
AgentType,
BuiltinAgentName,
_deep_merge,
)
from vibe.core.config import VibeConfig
from vibe.core.prompts import UtilityPrompt
from vibe.core.tools.base import ToolPermission
from vibe.core.types import LLMChunk, LLMMessage, LLMUsage, Role
class TestDeepMerge:
def test_simple_merge(self) -> None:
base = {"a": 1, "b": 2}
override = {"c": 3}
result = _deep_merge(base, override)
assert result == {"a": 1, "b": 2, "c": 3}
def test_override_existing_key(self) -> None:
base = {"a": 1, "b": 2}
override = {"b": 3}
result = _deep_merge(base, override)
assert result == {"a": 1, "b": 3}
def test_nested_dict_merge(self) -> None:
base = {"a": {"x": 1, "y": 2}}
override = {"a": {"y": 3, "z": 4}}
result = _deep_merge(base, override)
assert result == {"a": {"x": 1, "y": 3, "z": 4}}
def test_deeply_nested_merge(self) -> None:
base = {"a": {"b": {"c": 1}}}
override = {"a": {"b": {"d": 2}}}
result = _deep_merge(base, override)
assert result == {"a": {"b": {"c": 1, "d": 2}}}
def test_override_dict_with_non_dict(self) -> None:
base = {"a": {"x": 1}}
override = {"a": "replaced"}
result = _deep_merge(base, override)
assert result == {"a": "replaced"}
def test_override_non_dict_with_dict(self) -> None:
base = {"a": "string"}
override = {"a": {"x": 1}}
result = _deep_merge(base, override)
assert result == {"a": {"x": 1}}
def test_preserves_original_base(self) -> None:
base = {"a": 1, "b": {"c": 2}}
override = {"b": {"d": 3}}
_deep_merge(base, override)
assert base == {"a": 1, "b": {"c": 2}}
def test_empty_override(self) -> None:
base = {"a": 1, "b": 2}
override: dict = {}
result = _deep_merge(base, override)
assert result == {"a": 1, "b": 2}
def test_empty_base(self) -> None:
base: dict = {}
override = {"a": 1}
result = _deep_merge(base, override)
assert result == {"a": 1}
def test_lists_are_overridden_not_merged(self) -> None:
"""Lists should be replaced entirely, not merged element-by-element."""
base = {"tools": ["read", "grep", "bash"]}
override = {"tools": ["write_file"]}
result = _deep_merge(base, override)
assert result == {"tools": ["write_file"]}
def test_nested_lists_are_overridden_not_merged(self) -> None:
"""Nested lists in dicts should also be replaced, not merged."""
base = {"config": {"enabled_tools": ["a", "b", "c"], "other": 1}}
override = {"config": {"enabled_tools": ["x", "y"]}}
result = _deep_merge(base, override)
assert result == {"config": {"enabled_tools": ["x", "y"], "other": 1}}
class TestAgentSafety:
def test_safety_enum_values(self) -> None:
assert AgentSafety.SAFE == "safe"
assert AgentSafety.NEUTRAL == "neutral"
assert AgentSafety.DESTRUCTIVE == "destructive"
assert AgentSafety.YOLO == "yolo"
def test_default_agent_is_neutral(self) -> None:
assert BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].safety == AgentSafety.NEUTRAL
def test_auto_approve_agent_is_yolo(self) -> None:
assert BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].safety == AgentSafety.YOLO
def test_plan_agent_is_safe(self) -> None:
assert BUILTIN_AGENTS[BuiltinAgentName.PLAN].safety == AgentSafety.SAFE
def test_accept_edits_agent_is_destructive(self) -> None:
assert (
BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].safety
== AgentSafety.DESTRUCTIVE
)
class TestAgentProfile:
def test_all_builtin_agents_have_valid_names(self) -> None:
acp_only = {BuiltinAgentName.CHAT}
assert set(BUILTIN_AGENTS.keys()) == set(BuiltinAgentName) - acp_only
def test_display_name_property(self) -> None:
assert BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].display_name == "Default"
assert (
BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].display_name == "Auto Approve"
)
assert BUILTIN_AGENTS[BuiltinAgentName.PLAN].display_name == "Plan"
assert (
BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].display_name == "Accept Edits"
)
def test_description_property(self) -> None:
assert (
"approval" in BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].description.lower()
)
assert (
"auto" in BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].description.lower()
)
assert "read-only" in BUILTIN_AGENTS[BuiltinAgentName.PLAN].description.lower()
assert (
"edits" in BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].description.lower()
)
def test_explore_is_subagent(self) -> None:
assert BUILTIN_AGENTS[BuiltinAgentName.EXPLORE].agent_type == AgentType.SUBAGENT
def test_agents(self) -> None:
agents = [
name
for name, profile in BUILTIN_AGENTS.items()
if profile.agent_type == AgentType.AGENT
]
assert set(agents) == {
BuiltinAgentName.DEFAULT,
BuiltinAgentName.PLAN,
BuiltinAgentName.ACCEPT_EDITS,
BuiltinAgentName.AUTO_APPROVE,
BuiltinAgentName.LEAN,
}
class TestAgentApplyToConfig:
def test_profile_disabled_tools_are_merged_with_base_config(self) -> None:
base = VibeConfig(
include_project_context=False,
include_prompt_detail=False,
disabled_tools=["ask_user_question"],
)
result = BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].apply_to_config(base)
assert set(result.disabled_tools) == {"ask_user_question", "exit_plan_mode"}
def test_profile_disabled_tools_preserve_user_disabled_tools(self) -> None:
base = VibeConfig(
include_project_context=False,
include_prompt_detail=False,
disabled_tools=["ask_user_question", "custom_tool"],
)
result = BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].apply_to_config(base)
assert set(result.disabled_tools) == {
"ask_user_question",
"custom_tool",
"exit_plan_mode",
}
def test_base_disabled_tools_are_filtered_from_profile_enabled_tools(self) -> None:
base = VibeConfig(
include_project_context=False,
include_prompt_detail=False,
disabled_tools=["ask_user_question"],
)
result = CHAT.apply_to_config(base)
assert "ask_user_question" not in result.enabled_tools
assert "grep" in result.enabled_tools
assert "read" in result.enabled_tools
assert "task" in result.enabled_tools
def test_base_disabled_tools_filter_supports_glob_patterns(self) -> None:
base = VibeConfig(
include_project_context=False,
include_prompt_detail=False,
disabled_tools=["ask_*"],
)
agent = AgentProfile(
name="custom",
display_name="Custom",
description="",
safety=AgentSafety.NEUTRAL,
overrides={"enabled_tools": ["grep", "ask_user_question", "ask_extra"]},
)
result = agent.apply_to_config(base)
assert result.enabled_tools == ["grep"]
def test_empty_base_disabled_tools_leaves_enabled_tools_untouched(self) -> None:
base = VibeConfig(
include_project_context=False,
include_prompt_detail=False,
disabled_tools=[],
)
result = CHAT.apply_to_config(base)
assert "ask_user_question" in result.enabled_tools
def test_custom_prompt_found_in_global_when_missing_from_project(
self, mock_prompts_dirs: tuple[Path, Path]
) -> None:
_, global_prompts = mock_prompts_dirs
(global_prompts / "cc.md").write_text("Global custom prompt")
base = VibeConfig(include_project_context=False, include_prompt_detail=False)
agent = AgentProfile(
name="cc",
display_name="Cc",
description="",
safety=AgentSafety.NEUTRAL,
overrides={"system_prompt_id": "cc"},
)
result = agent.apply_to_config(base)
assert result.system_prompt_id == "cc"
assert result.system_prompt == "Global custom prompt"
def test_custom_prompt_overrides_builtin(
self, mock_prompts_dirs: tuple[Path, Path]
) -> None:
"""Custom prompts in .vibe/prompts/ should override built-in prompts.
A user-provided explore.md (or any built-in prompt name) in the
project or user prompts directory must take priority over the
bundled SystemPrompt enum.
"""
project_prompts, _ = mock_prompts_dirs
(project_prompts / "explore.md").write_text("My custom explore prompt")
config = VibeConfig(
system_prompt_id="explore",
include_project_context=False,
include_prompt_detail=False,
)
assert config.system_prompt == "My custom explore prompt"
def test_custom_compaction_prompt_found_in_global_when_missing_from_project(
self, mock_prompts_dirs: tuple[Path, Path]
) -> None:
_, global_prompts = mock_prompts_dirs
(global_prompts / "proofs.md").write_text("Global custom compaction prompt")
config = VibeConfig(
compaction_prompt_id="proofs",
include_project_context=False,
include_prompt_detail=False,
)
assert config.compaction_prompt == "Global custom compaction prompt"
def test_custom_compaction_prompt_overrides_builtin(
self, mock_prompts_dirs: tuple[Path, Path]
) -> None:
project_prompts, _ = mock_prompts_dirs
(project_prompts / "compact.md").write_text("My custom compact prompt")
config = VibeConfig(
compaction_prompt_id="compact",
include_project_context=False,
include_prompt_detail=False,
)
assert config.compaction_prompt == "My custom compact prompt"
def test_default_compaction_prompt_falls_back_to_builtin(
self, mock_prompts_dirs: tuple[Path, Path]
) -> None:
config = VibeConfig(include_project_context=False, include_prompt_detail=False)
assert config.compaction_prompt == UtilityPrompt.COMPACT.read()
def test_invalid_compaction_prompt_reports_setting_name(
self, mock_prompts_dirs: tuple[Path, Path]
) -> None:
project_prompts, user_prompts = mock_prompts_dirs
(project_prompts / "alpha.md").write_text("a")
(user_prompts / "beta.md").write_text("b")
with pytest.raises(ValueError) as exc_info:
VibeConfig(
compaction_prompt_id="unknown",
include_project_context=False,
include_prompt_detail=False,
)
error_text = str(exc_info.value)
assert "Invalid compaction_prompt_id value: 'unknown'" in error_text
assert 'available prompts ("compact")' in error_text
assert '(available: "alpha", "beta")' in error_text
@pytest.mark.parametrize(
"malicious_id",
["../../../etc/passwd", "..", ".", "subdir/compact", "back\\slash", ""],
)
def test_prompt_id_rejects_path_traversal(
self, mock_prompts_dirs: tuple[Path, Path], malicious_id: str
) -> None:
with pytest.raises(ValueError, match="must be a bare filename"):
VibeConfig(
compaction_prompt_id=malicious_id,
include_project_context=False,
include_prompt_detail=False,
)
class TestAgentProfileOverrides:
def test_default_agent_disables_exit_plan_mode(self) -> None:
overrides = BUILTIN_AGENTS[BuiltinAgentName.DEFAULT].overrides
assert "exit_plan_mode" in overrides.get("base_disabled", [])
def test_auto_approve_agent_sets_bypass_tool_permissions(self) -> None:
overrides = BUILTIN_AGENTS[BuiltinAgentName.AUTO_APPROVE].overrides
assert overrides.get("bypass_tool_permissions") is True
def test_plan_agent_restricts_tools(self) -> None:
overrides = BUILTIN_AGENTS[BuiltinAgentName.PLAN].overrides
assert "tools" in overrides
tools = overrides["tools"]
assert "write_file" in tools
assert "edit" in tools
assert tools["write_file"]["permission"] == "never"
assert tools["edit"]["permission"] == "never"
assert len(tools["write_file"]["allowlist"]) > 0
assert len(tools["edit"]["allowlist"]) > 0
def test_accept_edits_agent_sets_tool_permissions(self) -> None:
overrides = BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].overrides
assert "tools" in overrides
tools_config = overrides["tools"]
assert "write_file" in tools_config
assert "edit" in tools_config
assert tools_config["write_file"]["permission"] == "always"
assert tools_config["edit"]["permission"] == "always"
class TestAgentManagerCycling:
@pytest.fixture
def base_config(self) -> VibeConfig:
return build_test_vibe_config(
include_project_context=False, include_prompt_detail=False
)
@pytest.fixture
def backend(self) -> FakeBackend:
return FakeBackend([
LLMChunk(
message=LLMMessage(role=Role.assistant, content="Test response"),
usage=LLMUsage(prompt_tokens=10, completion_tokens=5),
)
])
def test_get_agent_order_includes_primary_agents(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
)
order = agent.agent_manager.get_agent_order()
assert len(order) == 4
assert BuiltinAgentName.DEFAULT in order
assert BuiltinAgentName.AUTO_APPROVE in order
assert BuiltinAgentName.PLAN in order
assert BuiltinAgentName.ACCEPT_EDITS in order
def test_next_agent_cycles_through_all(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
)
order = agent.agent_manager.get_agent_order()
current = agent.agent_manager.active_profile
visited = [current.name]
for _ in range(len(order) - 1):
current = agent.agent_manager.next_agent(current)
visited.append(current.name)
assert len(set(visited)) == len(order)
def test_next_agent_wraps_around(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
)
order = agent.agent_manager.get_agent_order()
last_profile = agent.agent_manager.get_agent(order[-1])
first_profile = agent.agent_manager.get_agent(order[0])
assert agent.agent_manager.next_agent(last_profile).name == first_profile.name
class TestAgentProfileConfig:
def test_agent_profile_frozen(self) -> None:
profile = AgentProfile(
name="test",
display_name="Test",
description="Test agent",
safety=AgentSafety.NEUTRAL,
)
with pytest.raises(AttributeError):
profile.name = "changed" # pyright: ignore[reportAttributeAccessIssue]
class TestAgentSwitchAgent:
@pytest.fixture
def base_config(self) -> VibeConfig:
return build_test_vibe_config(
include_project_context=False, include_prompt_detail=False
)
@pytest.fixture
def backend(self) -> FakeBackend:
return FakeBackend([
LLMChunk(
message=LLMMessage(role=Role.assistant, content="Test response"),
usage=LLMUsage(prompt_tokens=10, completion_tokens=5),
)
])
@pytest.mark.asyncio
async def test_switch_to_plan_agent_has_tools_with_restricted_permissions(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
)
await agent.switch_agent(BuiltinAgentName.PLAN)
plan_tool_names = set(agent.tool_manager.available_tools.keys())
# Plan mode now has all tools available but with restricted permissions
assert "write_file" in plan_tool_names
assert "edit" in plan_tool_names
assert "grep" in plan_tool_names
assert "read" in plan_tool_names
assert agent.agent_profile.name == BuiltinAgentName.PLAN
# Verify write tools have "never" base permission
write_config = agent.tool_manager.get_tool_config("write_file")
assert write_config.permission == ToolPermission.NEVER
@pytest.mark.asyncio
async def test_switch_from_plan_to_default_restores_tools(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.PLAN, backend=backend
)
await agent.switch_agent(BuiltinAgentName.DEFAULT)
# Write tools should revert to default ASK permission
write_config = agent.tool_manager.get_tool_config("write_file")
assert write_config.permission == ToolPermission.ASK
assert agent.agent_profile.name == BuiltinAgentName.DEFAULT
@pytest.mark.asyncio
async def test_switch_agent_preserves_conversation_history(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
)
user_msg = LLMMessage(role=Role.user, content="Hello")
assistant_msg = LLMMessage(role=Role.assistant, content="Hi there")
agent.messages.append(user_msg)
agent.messages.append(assistant_msg)
await agent.switch_agent(BuiltinAgentName.PLAN)
assert len(agent.messages) == 3 # system + user + assistant
assert agent.messages[1].content == "Hello"
assert agent.messages[2].content == "Hi there"
@pytest.mark.asyncio
async def test_switch_to_same_agent_is_noop(
self, base_config: VibeConfig, backend: FakeBackend
) -> None:
agent = build_test_agent_loop(
config=base_config, agent_name=BuiltinAgentName.DEFAULT, backend=backend
)
original_config = agent.config
await agent.switch_agent(BuiltinAgentName.DEFAULT)
assert agent.config is original_config
assert agent.agent_profile.name == BuiltinAgentName.DEFAULT
class TestAcceptEditsAgent:
def test_accept_edits_config_sets_write_file_always(self) -> None:
overrides = BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].overrides
assert overrides["tools"]["write_file"]["permission"] == "always"
def test_accept_edits_config_sets_edit_always(self) -> None:
overrides = BUILTIN_AGENTS[BuiltinAgentName.ACCEPT_EDITS].overrides
assert overrides["tools"]["edit"]["permission"] == "always"
@pytest.mark.asyncio
async def test_accept_edits_agent_auto_approves_write_file(self) -> None:
backend = FakeBackend([])
config = build_test_vibe_config(enabled_tools=["write_file"])
agent = build_test_agent_loop(
config=config, agent_name=BuiltinAgentName.ACCEPT_EDITS, backend=backend
)
perm = agent.tool_manager.get_tool_config("write_file").permission
assert perm == ToolPermission.ALWAYS
@pytest.mark.asyncio
async def test_accept_edits_agent_requires_approval_for_other_tools(self) -> None:
backend = FakeBackend([])
config = build_test_vibe_config(enabled_tools=["bash"])
agent = build_test_agent_loop(
config=config, agent_name=BuiltinAgentName.ACCEPT_EDITS, backend=backend
)
perm = agent.tool_manager.get_tool_config("bash").permission
assert perm == ToolPermission.ASK
class TestPlanAgentToolRestriction:
@pytest.mark.asyncio
async def test_plan_agent_has_all_tools_with_restricted_write_permissions(
self,
) -> None:
backend = FakeBackend([
LLMChunk(
message=LLMMessage(role=Role.assistant, content="ok"),
usage=LLMUsage(prompt_tokens=10, completion_tokens=5),
)
])
config = build_test_vibe_config()
agent = build_test_agent_loop(
config=config, agent_name=BuiltinAgentName.PLAN, backend=backend
)
tool_names = set(agent.tool_manager.available_tools.keys())
# Plan mode now has all tools available
assert "grep" in tool_names
assert "read" in tool_names
assert "write_file" in tool_names
assert "edit" in tool_names
# But write tools have restricted permissions
write_config = agent.tool_manager.get_tool_config("write_file")
assert write_config.permission == ToolPermission.NEVER
assert len(write_config.allowlist) > 0
edit_config = agent.tool_manager.get_tool_config("edit")
assert edit_config.permission == ToolPermission.NEVER
assert len(edit_config.allowlist) > 0
class TestAgentManagerFiltering:
def test_enabled_agents_filters_to_only_enabled(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
enabled_agents=["default", "plan"],
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert len(agents) < len(manager._discovered)
assert "default" in agents
assert "plan" in agents
assert "auto-approve" not in agents
assert "accept-edits" not in agents
def test_disabled_agents_excludes_disabled(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
disabled_agents=["auto-approve", "accept-edits"],
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert len(agents) < len(manager._discovered)
assert "default" in agents
assert "plan" in agents
assert "auto-approve" not in agents
assert "accept-edits" not in agents
def test_enabled_agents_takes_precedence_over_disabled(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
enabled_agents=["default"],
disabled_agents=["default"], # Should be ignored
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert len(agents) == 1
assert "default" in agents
def test_glob_pattern_matching(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
disabled_agents=["auto-*", "accept-*"],
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert "default" in agents
assert "plan" in agents
assert "auto-approve" not in agents
assert "accept-edits" not in agents
def test_regex_pattern_matching(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
enabled_agents=["re:^(default|plan)$"],
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert len(agents) == 2
assert "default" in agents
assert "plan" in agents
def test_empty_enabled_agents_returns_all(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
enabled_agents=[],
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert "default" in agents
assert "plan" in agents
assert "auto-approve" in agents
assert "explore" in agents
def test_install_required_agents_hidden_by_default(self) -> None:
config = build_test_vibe_config(
include_project_context=False, include_prompt_detail=False
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert "lean" not in agents
def test_install_required_agents_visible_when_installed(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
installed_agents=["lean"],
)
manager = AgentManager(lambda: config)
agents = manager.available_agents
assert "lean" in agents
def test_get_subagents_respects_filtering(self) -> None:
config = build_test_vibe_config(
include_project_context=False,
include_prompt_detail=False,
disabled_agents=["explore"],
)
manager = AgentManager(lambda: config)
subagents = manager.get_subagents()
names = [a.name for a in subagents]
assert "explore" not in names
class TestAgentLoopInitialization:
def test_agent_system_prompt_id_is_applied_on_init(
self, mock_prompts_dirs: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch
) -> None:
_, global_prompts = mock_prompts_dirs
custom_prompt_content = "CUSTOM_AGENT_PROMPT_MARKER"
(global_prompts / "custom_agent.md").write_text(custom_prompt_content)
custom_agent = AgentProfile(
name="custom_test_agent",
display_name="Custom Test",
description="Test agent with custom system prompt",
safety=AgentSafety.NEUTRAL,
overrides={"system_prompt_id": "custom_agent"},
)
patched_agents = {**BUILTIN_AGENTS, "custom_test_agent": custom_agent}
monkeypatch.setattr("vibe.core.agents.models.BUILTIN_AGENTS", patched_agents)
monkeypatch.setattr("vibe.core.agents.manager.BUILTIN_AGENTS", patched_agents)
config = build_test_vibe_config(
include_project_context=False, include_prompt_detail=False
)
assert config.system_prompt_id == "cli", (
"Base config should use default 'cli' prompt"
)
agent_loop = build_test_agent_loop(
config=config, agent_name="custom_test_agent"
)
assert agent_loop.config.system_prompt_id == "custom_agent", (
"Merged config should have the agent's system_prompt_id override"
)
system_message = agent_loop.messages[0]
assert system_message.role == Role.system
assert system_message.content is not None
assert custom_prompt_content in system_message.content, (
f"System message should contain custom prompt content. "
f"Expected '{custom_prompt_content}' to be in system message."
)
class TestActConsumersUseAclosing:
def test_no_bare_async_for_over_act(self) -> None:
vibe_pkg = TESTS_ROOT.parent / "vibe"
violations: list[str] = []
for path in vibe_pkg.rglob("*.py"):
tree = ast.parse(path.read_text(), filename=str(path))
for node in ast.walk(tree):
if not isinstance(node, ast.AsyncFor):
continue
match node.iter:
case ast.Call(func=ast.Attribute(attr="act")):
violations.append(f"{path}:{node.lineno}")
assert not violations, (
"Bare `async for ... in .act()` found — wrap in "
"contextlib.aclosing(). See issue #569.\n" + "\n".join(violations)
)

View file

@ -0,0 +1,122 @@
from __future__ import annotations
from pathlib import Path
import tomllib
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from vibe.core.tools.base import ToolPermission
from vibe.core.tools.permissions import PermissionScope, RequiredPermission
def _read_persisted_config(config_dir: Path) -> dict:
config_file = config_dir / "config.toml"
with config_file.open("rb") as f:
return tomllib.load(f)
class TestApproveAlwaysPermanentNoGranularPermissions:
def test_sets_tool_permission_always_in_config(self, config_dir: Path):
agent = build_test_agent_loop()
agent.approve_always("bash", None, save_permanently=True)
persisted = _read_persisted_config(config_dir)
assert persisted["tools"]["bash"]["permission"] == "always"
def test_session_only_does_not_persist(self, config_dir: Path):
agent = build_test_agent_loop()
agent.approve_always("bash", None, save_permanently=False)
assert (
agent.tool_manager.get_tool_config("bash").permission
== ToolPermission.ALWAYS
)
persisted = _read_persisted_config(config_dir)
assert "bash" not in persisted.get("tools", {})
class TestApproveAlwaysPermanentWithGranularPermissions:
def _make_permissions(self) -> list[RequiredPermission]:
return [
RequiredPermission(
scope=PermissionScope.COMMAND_PATTERN,
invocation_pattern="npm install foo",
session_pattern="npm install *",
label="npm install *",
)
]
def test_persists_allowlist_to_config(self, config_dir: Path):
agent = build_test_agent_loop()
perms = self._make_permissions()
agent.approve_always("bash", perms, save_permanently=True)
persisted = _read_persisted_config(config_dir)
assert persisted["tools"]["bash"]["allowlist"] == ["npm install"]
def test_also_adds_session_rules(self, config_dir: Path):
agent = build_test_agent_loop()
perms = self._make_permissions()
agent.approve_always("bash", perms, save_permanently=True)
assert len(agent._permission_store._rules) == 1
rule = agent._permission_store._rules[0]
assert rule.tool_name == "bash"
assert rule.scope == PermissionScope.COMMAND_PATTERN
assert rule.session_pattern == "npm install *"
def test_session_only_does_not_persist_allowlist(self, config_dir: Path):
agent = build_test_agent_loop()
perms = self._make_permissions()
agent.approve_always("bash", perms, save_permanently=False)
assert len(agent._permission_store._rules) == 1
persisted = _read_persisted_config(config_dir)
assert "bash" not in persisted.get("tools", {})
def test_does_not_duplicate_existing_allowlist_entries(self, config_dir: Path):
config = build_test_vibe_config(tools={"bash": {"allowlist": ["npm install"]}})
agent = build_test_agent_loop(config=config)
perms = self._make_permissions()
agent.approve_always("bash", perms, save_permanently=True)
persisted = _read_persisted_config(config_dir)
# Pattern already existed -- nothing new should be written
assert persisted.get("tools", {}).get("bash", {}).get("allowlist") is None
def test_appends_new_patterns_to_existing_allowlist(self, config_dir: Path):
config = build_test_vibe_config(tools={"bash": {"allowlist": ["git"]}})
agent = build_test_agent_loop(config=config)
perms = self._make_permissions()
agent.approve_always("bash", perms, save_permanently=True)
persisted = _read_persisted_config(config_dir)
assert persisted["tools"]["bash"]["allowlist"] == ["git", "npm install"]
def test_multiple_permissions_persisted(self, config_dir: Path):
agent = build_test_agent_loop()
perms = [
RequiredPermission(
scope=PermissionScope.COMMAND_PATTERN,
invocation_pattern="npm install foo",
session_pattern="npm install *",
label="npm install *",
),
RequiredPermission(
scope=PermissionScope.OUTSIDE_DIRECTORY,
invocation_pattern="/tmp/newdir",
session_pattern="/tmp/*",
label="/tmp/*",
),
]
agent.approve_always("bash", perms, save_permanently=True)
persisted = _read_persisted_config(config_dir)
assert persisted["tools"]["bash"]["allowlist"] == ["/tmp/*", "npm install"]

View file

@ -0,0 +1,593 @@
"""Tests for deferred initialization: _complete_init, _wait_for_init, integrate_mcp idempotency."""
from __future__ import annotations
import asyncio
import threading
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.mock.utils import mock_llm_chunk
from tests.stubs.fake_backend import FakeBackend
from tests.stubs.fake_connector_registry import FakeConnectorRegistry
from tests.stubs.fake_mcp_registry import FakeMCPRegistry
from vibe.core import agent_loop as agent_loop_module
from vibe.core.agent_loop import AgentLoop
from vibe.core.config import MCPStdio
from vibe.core.telemetry.types import TerminalEmulator
from vibe.core.tools.manager import ToolManager
from vibe.core.tools.mcp import AuthStatus
from vibe.core.tools.mcp.tools import RemoteTool
def _build_uninitiated_loop(**kwargs):
"""Build a test loop with defer_heavy_init=True but without auto-starting the init thread."""
with patch.object(AgentLoop, "_start_deferred_init"):
return build_test_agent_loop(defer_heavy_init=True, **kwargs)
# ---------------------------------------------------------------------------
# _complete_init
# ---------------------------------------------------------------------------
def _run_init(loop: AgentLoop) -> None:
"""Run _complete_init in a thread (matching production behavior) and wait."""
thread = threading.Thread(target=loop._complete_init, daemon=True)
loop._deferred_init_thread = thread
thread.start()
thread.join()
class TestCompleteInit:
def test_success_sets_init_complete(self) -> None:
loop = _build_uninitiated_loop()
assert not loop.is_initialized
_run_init(loop)
assert loop.is_initialized
assert loop._init_error is None
def test_failure_sets_init_complete_and_stores_error(self) -> None:
loop = _build_uninitiated_loop()
error = RuntimeError("mcp boom")
with patch.object(loop.tool_manager, "integrate_all", side_effect=error):
_run_init(loop)
assert loop.is_initialized
assert loop._init_error is error
def test_mcp_failure_sets_init_error(self) -> None:
mcp_server = MCPStdio(name="test-server", transport="stdio", command="echo")
config = build_test_vibe_config(mcp_servers=[mcp_server])
loop = _build_uninitiated_loop(config=config)
with patch.object(
loop.tool_manager._mcp_registry,
"get_tools_async",
side_effect=RuntimeError("mcp discovery boom"),
):
_run_init(loop)
assert loop.is_initialized
assert isinstance(loop._init_error, RuntimeError)
assert str(loop._init_error) == "mcp discovery boom"
# ---------------------------------------------------------------------------
# wait_until_ready
# ---------------------------------------------------------------------------
class TestWaitForInit:
@pytest.mark.asyncio
async def test_returns_immediately_when_already_complete(self) -> None:
loop = build_test_agent_loop(defer_heavy_init=True)
await loop.wait_until_ready() # should not block
assert loop.is_initialized
@pytest.mark.asyncio
async def test_waits_for_background_thread(self) -> None:
loop = build_test_agent_loop(defer_heavy_init=True)
await loop.wait_until_ready()
assert loop.is_initialized
@pytest.mark.asyncio
async def test_raises_stored_error(self) -> None:
loop = _build_uninitiated_loop()
error = RuntimeError("init failed")
with patch.object(loop.tool_manager, "integrate_all", side_effect=error):
loop._complete_init()
with pytest.raises(RuntimeError, match="init failed"):
await loop.wait_until_ready()
@pytest.mark.asyncio
async def test_raises_error_for_every_caller(self) -> None:
loop = _build_uninitiated_loop()
error = RuntimeError("once only")
with patch.object(loop.tool_manager, "integrate_all", side_effect=error):
loop._complete_init()
with pytest.raises(RuntimeError):
await loop.wait_until_ready()
with pytest.raises(RuntimeError):
await loop.wait_until_ready()
# ---------------------------------------------------------------------------
# integrate_mcp idempotency
# ---------------------------------------------------------------------------
class TestIntegrateMcpIdempotency:
def test_second_call_is_noop(self) -> None:
mcp_server = MCPStdio(name="test-server", transport="stdio", command="echo")
config = build_test_vibe_config(mcp_servers=[mcp_server])
registry = FakeMCPRegistry()
manager = ToolManager(lambda: config, mcp_registry=registry, defer_mcp=True)
manager.integrate_mcp()
tools_after_first = dict(manager.registered_tools)
# Spy on the registry to ensure get_tools is not called again.
registry.get_tools = MagicMock(wraps=registry.get_tools)
manager.integrate_mcp()
registry.get_tools.assert_not_called()
assert manager.registered_tools == tools_after_first
def test_flag_not_set_when_no_servers(self) -> None:
config = build_test_vibe_config(mcp_servers=[])
manager = ToolManager(lambda: config, defer_mcp=True)
manager.integrate_mcp()
# No servers means the method returns early without setting the flag,
# so a future call with servers would still run discovery.
assert not manager._mcp_integrated
def test_no_servers_syncs_shared_registry_status(self) -> None:
config = build_test_vibe_config(
mcp_servers=[MCPStdio(name="srv", transport="stdio", command="echo")]
)
registry = FakeMCPRegistry()
manager = ToolManager(lambda: config, mcp_registry=registry, defer_mcp=True)
manager.integrate_mcp()
assert registry.status() == {"srv": AuthStatus.STDIO}
config = build_test_vibe_config(mcp_servers=[])
manager = ToolManager(lambda: config, mcp_registry=registry, defer_mcp=True)
manager.integrate_mcp()
assert registry.status() == {}
assert not manager._mcp_integrated
class TestRefreshRemoteTools:
@pytest.mark.asyncio
async def test_refresh_rediscovers_mcp_and_connector_tools(self) -> None:
mcp_server = MCPStdio(name="srv", transport="stdio", command="echo")
config = build_test_vibe_config(mcp_servers=[mcp_server])
registry = FakeMCPRegistry()
registry.get_tools_async = AsyncMock(wraps=registry.get_tools_async)
connector_registry = FakeConnectorRegistry({
"alpha": [RemoteTool(name="search", description="Search alpha")]
})
manager = ToolManager(
lambda: config,
mcp_registry=registry,
connector_registry=connector_registry,
defer_mcp=True,
)
await manager.refresh_remote_tools_async()
assert "srv_fake_tool" in manager.registered_tools
assert "connector_alpha_search" in manager.registered_tools
connector_registry._fake_connectors = {
"beta": [RemoteTool(name="list", description="List beta")]
}
await manager.refresh_remote_tools_async()
assert registry.get_tools_async.await_count == 2
assert "srv_fake_tool" in manager.registered_tools
assert "connector_alpha_search" not in manager.registered_tools
assert "connector_beta_list" in manager.registered_tools
class TestDeferredInitPublicMethods:
@pytest.mark.asyncio
async def test_act_waits_for_deferred_init(self) -> None:
loop = build_test_agent_loop(
defer_heavy_init=True, backend=FakeBackend(mock_llm_chunk(content="hello"))
)
events = [event async for event in loop.act("Hello")]
assert loop.is_initialized
assert [event.content for event in events if hasattr(event, "content")][
-1
] == "hello"
@pytest.mark.asyncio
async def test_reload_with_initial_messages_waits_for_deferred_init(self) -> None:
loop = build_test_agent_loop(defer_heavy_init=True)
await loop.reload_with_initial_messages()
assert loop.is_initialized
@pytest.mark.asyncio
async def test_switch_agent_waits_for_deferred_init(self) -> None:
loop = build_test_agent_loop(defer_heavy_init=True)
await loop.switch_agent("plan")
assert loop.is_initialized
assert loop.agent_profile.name == "plan"
@pytest.mark.asyncio
async def test_clear_history_waits_for_deferred_init(self) -> None:
loop = build_test_agent_loop(
defer_heavy_init=True, backend=FakeBackend(mock_llm_chunk(content="hello"))
)
[_ async for _ in loop.act("Hello")]
await loop.clear_history()
assert loop.is_initialized
assert len(loop.messages) == 1
@pytest.mark.asyncio
async def test_compact_waits_for_deferred_init(self) -> None:
loop = build_test_agent_loop(
defer_heavy_init=True,
backend=FakeBackend([
[mock_llm_chunk(content="hello")],
[mock_llm_chunk(content="summary")],
]),
)
[_ async for _ in loop.act("Hello")]
summary = await loop.compact()
assert loop.is_initialized
assert summary == "summary"
@pytest.mark.asyncio
async def test_inject_user_context_waits_for_deferred_init(self) -> None:
loop = build_test_agent_loop(defer_heavy_init=True)
await loop.inject_user_context("context")
assert loop.is_initialized
assert loop.messages[-1].content == "context"
# ---------------------------------------------------------------------------
# start_initialize_experiments / wait_until_ready experiment gating
# ---------------------------------------------------------------------------
class TestStartInitializeExperiments:
@pytest.mark.asyncio
async def test_does_not_block_caller(self) -> None:
loop = build_test_agent_loop()
gate = asyncio.Event()
async def slow_init() -> None:
await gate.wait()
with patch.object(loop, "initialize_experiments", side_effect=slow_init):
loop.start_initialize_experiments()
task = loop._experiments_task
assert task is not None
assert not task.done()
gate.set()
await task
@pytest.mark.asyncio
async def test_is_idempotent(self) -> None:
loop = build_test_agent_loop()
init_mock = AsyncMock()
with patch.object(loop, "initialize_experiments", new=init_mock):
loop.start_initialize_experiments()
first_task = loop._experiments_task
loop.start_initialize_experiments()
second_task = loop._experiments_task
assert first_task is second_task
assert first_task is not None
await first_task
assert init_mock.await_count == 1
@pytest.mark.asyncio
async def test_sets_pending_telemetry_flags(self) -> None:
loop = build_test_agent_loop()
with patch.object(loop, "initialize_experiments", new=AsyncMock()):
loop.start_initialize_experiments()
assert loop._pending_new_session_telemetry is True
assert loop._ready_telemetry_pending is True
task = loop._experiments_task
assert task is not None
await task
@pytest.mark.asyncio
async def test_refreshes_system_prompt_when_experiments_update(self) -> None:
loop = build_test_agent_loop(terminal_emulator=TerminalEmulator.VSCODE)
refresh_mock = AsyncMock()
init_mock = AsyncMock(return_value=True)
with (
patch.object(
agent_loop_module, "session_initialize_experiments", new=init_mock
),
patch.object(loop, "refresh_system_prompt", new=refresh_mock),
):
await loop.initialize_experiments()
refresh_mock.assert_awaited_once()
init_mock.assert_awaited_once()
init_args = init_mock.await_args
assert init_args is not None
assert init_args.kwargs["terminal_emulator"] is TerminalEmulator.VSCODE
def test_new_session_telemetry_uses_provided_terminal_emulator(self) -> None:
loop = build_test_agent_loop(terminal_emulator=TerminalEmulator.VSCODE)
send_new_session = MagicMock()
with patch.object(
loop.telemetry_client, "send_new_session", new=send_new_session
):
loop.emit_new_session_telemetry()
assert (
send_new_session.call_args.kwargs["terminal_emulator"]
is TerminalEmulator.VSCODE
)
@pytest.mark.asyncio
async def test_does_not_refresh_system_prompt_when_experiments_unchanged(
self,
) -> None:
loop = build_test_agent_loop()
refresh_mock = AsyncMock()
with (
patch.object(
agent_loop_module,
"session_initialize_experiments",
new=AsyncMock(return_value=False),
),
patch.object(loop, "refresh_system_prompt", new=refresh_mock),
):
await loop.initialize_experiments()
refresh_mock.assert_not_awaited()
class TestWaitUntilReadyJoinsExperiments:
@pytest.mark.asyncio
async def test_joins_in_flight_task(self) -> None:
loop = build_test_agent_loop()
gate = asyncio.Event()
completed = False
async def slow_init() -> None:
nonlocal completed
await gate.wait()
completed = True
with patch.object(loop, "initialize_experiments", side_effect=slow_init):
loop.start_initialize_experiments()
async def release() -> None:
await asyncio.sleep(0.01)
gate.set()
asyncio.create_task(release())
await loop.wait_until_ready()
assert completed is True
task = loop._experiments_task
assert task is not None
assert task.done()
@pytest.mark.asyncio
async def test_emits_new_session_telemetry_once(self) -> None:
loop = build_test_agent_loop()
emit_new_session = MagicMock()
with (
patch.object(loop, "initialize_experiments", new=AsyncMock()),
patch.object(loop, "emit_new_session_telemetry", new=emit_new_session),
):
loop.start_initialize_experiments()
await loop.wait_until_ready()
await loop.wait_until_ready()
emit_new_session.assert_called_once()
assert loop._pending_new_session_telemetry is False
@pytest.mark.asyncio
async def test_emits_ready_telemetry_when_only_experiments_deferred(self) -> None:
loop = build_test_agent_loop()
emit_ready = MagicMock()
with (
patch.object(loop, "initialize_experiments", new=AsyncMock()),
patch.object(loop, "emit_ready_telemetry", new=emit_ready),
):
loop.start_initialize_experiments()
await loop.wait_until_ready()
await loop.wait_until_ready()
emit_ready.assert_called_once()
((duration,), _) = emit_ready.call_args
assert isinstance(duration, int)
assert duration >= 0
assert loop._ready_telemetry_pending is False
@pytest.mark.asyncio
async def test_does_not_emit_new_session_when_only_hydrating(self) -> None:
loop = build_test_agent_loop()
emit_new_session = MagicMock()
with (
patch.object(loop, "hydrate_experiments_from_session", new=AsyncMock()),
patch.object(loop, "emit_new_session_telemetry", new=emit_new_session),
):
await loop.hydrate_experiments_from_session()
await loop.wait_until_ready()
emit_new_session.assert_not_called()
assert loop._pending_new_session_telemetry is False
@pytest.mark.asyncio
async def test_no_op_when_nothing_deferred(self) -> None:
loop = build_test_agent_loop()
emit_ready = MagicMock()
emit_new_session = MagicMock()
with (
patch.object(loop, "emit_ready_telemetry", new=emit_ready),
patch.object(loop, "emit_new_session_telemetry", new=emit_new_session),
):
await loop.wait_until_ready()
emit_ready.assert_not_called()
emit_new_session.assert_not_called()
class TestACloseCancelsExperimentsTask:
@pytest.mark.asyncio
async def test_cancels_in_flight_task(self) -> None:
loop = build_test_agent_loop()
gate = asyncio.Event()
async def never_completing() -> None:
await gate.wait()
with patch.object(loop, "initialize_experiments", side_effect=never_completing):
loop.start_initialize_experiments()
task = loop._experiments_task
assert task is not None
assert not task.done()
await loop.aclose()
assert task.done()
assert task.cancelled()
@pytest.mark.asyncio
async def test_does_not_cancel_completed_task(self) -> None:
loop = build_test_agent_loop()
with patch.object(loop, "initialize_experiments", new=AsyncMock()):
loop.start_initialize_experiments()
task = loop._experiments_task
assert task is not None
await task
await loop.aclose()
assert task.done()
assert not task.cancelled()
class TestCycleAgentDuringInit:
@pytest.mark.asyncio
async def test_shift_tab_during_experiments_init_does_not_crash(self) -> None:
"""Regression: shift+tab during init crashed with
RuntimeError("await wasn't used with future").
_cycle_agent ran switch_agent via asyncio.run() in a thread worker,
creating a second event loop. wait_until_ready then tried to await
_experiments_task (owned by the main Textual loop) from that new loop.
The fix uses asyncio.run_coroutine_threadsafe() to schedule on the
main loop instead. This test presses shift+tab while experiments
are still initializing and asserts the worker completes without error.
"""
from tests.conftest import build_test_vibe_app
gate = asyncio.Event()
async def slow_init() -> None:
await gate.wait()
agent_loop = build_test_agent_loop()
app = build_test_vibe_app(agent_loop=agent_loop)
async with app.run_test() as pilot:
with patch.object(
agent_loop, "initialize_experiments", side_effect=slow_init
):
agent_loop._experiments_task = None # reset so start_ re-fires
agent_loop.start_initialize_experiments()
assert agent_loop._experiments_task is not None
assert not agent_loop._experiments_task.done()
# Press shift+tab while experiments are still running.
await pilot.press("shift+tab")
await pilot.pause(0.05)
# Unblock experiments so switch_agent can complete.
gate.set()
# wait_for_complete raises WorkerFailed if the thread worker
# crashed — which is exactly what happened before the fix.
await pilot.app.workers.wait_for_complete()
assert agent_loop.agent_profile.name == "plan"
class TestActGatesOnExperiments:
@pytest.mark.asyncio
async def test_act_awaits_experiments_before_llm_call(self) -> None:
loop = build_test_agent_loop(
backend=FakeBackend(mock_llm_chunk(content="hello"))
)
gate = asyncio.Event()
finished_init = False
async def slow_init() -> None:
nonlocal finished_init
await gate.wait()
finished_init = True
with patch.object(loop, "initialize_experiments", side_effect=slow_init):
loop.start_initialize_experiments()
async def release() -> None:
await asyncio.sleep(0.01)
gate.set()
asyncio.create_task(release())
events = [event async for event in loop.act("Hello")]
assert finished_init is True
assert any(getattr(event, "content", None) == "hello" for event in events)