Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Hdandria <henri.dandria@mistral.ai>
Co-authored-by: Ivana Dunisijevic <ivana.dunisijevic@mistral.ai>
Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Mert Unsal <mert.unsal@mistral.ai>
Co-authored-by: Michel Thomazo <51709227+michelTho@users.noreply.github.com>
Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Val <102326092+vdeva@users.noreply.github.com>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: renovate-mistral[bot] <253709520+renovate-mistral[bot]@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Clément Drouin 2026-06-19 11:01:24 +02:00 committed by GitHub
parent 564a14365e
commit 6bedf271ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
223 changed files with 10533 additions and 6947 deletions

View file

View file

@ -0,0 +1,181 @@
from __future__ import annotations
from collections.abc import Iterator
import json
from typing import Any
import httpx
import pytest
import respx
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.constants import (
ANTHROPIC_BASE_URL,
ANTHROPIC_MESSAGES_PATH,
CHAT_COMPLETIONS_PATH,
CONNECTORS_BOOTSTRAP_PATH,
MISTRAL_BASE_URL,
OPENAI_BASE_URL,
OPENAI_RESPONSES_PATH,
)
from vibe.core.agent_loop import AgentLoop
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.llm.backend.factory import BACKEND_FACTORY
from vibe.core.types import AssistantEvent, Backend, BaseEvent
def assistant_text(events: list[BaseEvent]) -> str:
return "".join(
e.content for e in events if isinstance(e, AssistantEvent) and e.content
).strip()
@pytest.fixture(autouse=True)
def _git_offline(monkeypatch: pytest.MonkeyPatch) -> None:
# Restrict git to the local file protocol so no test can fetch/push over the
# network, even if a remote is misconfigured to a real URL.
monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file")
def e2e_config(**overrides: Any) -> VibeConfig:
# Defaults give the mistral provider/model (api.mistral.ai, the respx-mocked
# base). The default model reasons, but that only adds reasoning_effort to the
# request, which the mocked responses ignore.
# The e2e harness mocks the connector bootstrap endpoint via respx, so
# connector discovery is enabled (and fast) here unlike the unit-test default.
return build_test_vibe_config(
enabled_tools=overrides.pop("enabled_tools", []),
enable_connectors=overrides.pop("enable_connectors", True),
system_prompt_id="tests",
include_project_context=False,
include_prompt_detail=False,
**overrides,
)
def _generic_e2e_config(
*, name: str, api_base: str, api_style: str, model: str, **overrides: Any
) -> VibeConfig:
provider = ProviderConfig(
name=name,
api_base=api_base,
api_key_env_var=f"{name.upper()}_API_KEY",
api_style=api_style,
backend=Backend.GENERIC,
)
models = [ModelConfig(name=model, provider=name, alias=name)]
return e2e_config(
active_model=name, models=models, providers=[provider], **overrides
)
def anthropic_e2e_config(**overrides: Any) -> VibeConfig:
return _generic_e2e_config(
name="anthropic",
api_base=ANTHROPIC_BASE_URL,
api_style="anthropic",
model="claude-test",
**overrides,
)
def openai_responses_e2e_config(**overrides: Any) -> VibeConfig:
return _generic_e2e_config(
name="openai",
api_base=f"{OPENAI_BASE_URL}/v1",
api_style="openai-responses",
model="gpt-test",
**overrides,
)
@pytest.fixture
def mock_mistral() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=MISTRAL_BASE_URL, assert_all_called=False) as router:
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
return_value=httpx.Response(200, json={"connectors": []})
)
yield router
def build_e2e_agent_loop(
*, config: VibeConfig | None = None, enable_streaming: bool = False, **kwargs: Any
) -> AgentLoop:
resolved_config = config or e2e_config()
provider = resolved_config.providers[0]
# todo now we use build_test_agent_loop for the fake mcp registry.
# Maybe mock http tool calls and instantiate a real registry later for more coverage
return build_test_agent_loop(
config=resolved_config,
agent_name=kwargs.pop("agent_name", BuiltinAgentName.AUTO_APPROVE),
backend=BACKEND_FACTORY[provider.backend](provider=provider),
enable_streaming=enable_streaming,
**kwargs,
)
class ProviderAPI:
"""Stubs a provider's completion wire; the AgentLoop stays the subject."""
def __init__(self, router: respx.MockRouter, path: str) -> None:
self.route = router.post(path)
def reply(self, *completions: dict[str, Any]) -> None:
responses = [httpx.Response(200, json=c) for c in completions]
if len(responses) == 1:
self.route.mock(return_value=responses[0])
else:
self.route.mock(side_effect=responses)
def reply_stream(self, chunks: list[bytes]) -> None:
self.route.mock(return_value=self._stream_response(chunks))
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
self.route.mock(
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
)
@staticmethod
def _stream_response(chunks: list[bytes]) -> httpx.Response:
return httpx.Response(
200,
stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
headers={"Content-Type": "text/event-stream"},
)
@property
def request_json(self) -> dict[str, Any]:
return json.loads(self.route.calls.last.request.content)
class MistralAPI(ProviderAPI):
def __init__(self, router: respx.MockRouter) -> None:
super().__init__(router, CHAT_COMPLETIONS_PATH)
@pytest.fixture
def mistral_api(mock_mistral: respx.MockRouter) -> MistralAPI:
return MistralAPI(mock_mistral)
@pytest.fixture
def mock_anthropic() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=ANTHROPIC_BASE_URL, assert_all_called=False) as router:
yield router
@pytest.fixture
def anthropic_api(mock_anthropic: respx.MockRouter) -> ProviderAPI:
return ProviderAPI(mock_anthropic, ANTHROPIC_MESSAGES_PATH)
@pytest.fixture
def mock_openai() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=OPENAI_BASE_URL, assert_all_called=False) as router:
yield router
@pytest.fixture
def openai_responses_api(mock_openai: respx.MockRouter) -> ProviderAPI:
return ProviderAPI(mock_openai, OPENAI_RESPONSES_PATH)

View file

@ -0,0 +1,111 @@
from __future__ import annotations
from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import (
MistralAPI,
assistant_text,
build_e2e_agent_loop,
e2e_config,
)
from tests.backend.data.mistral import (
STREAMED_SIMPLE_CONVERSATION_PARAMS,
mistral_completion,
)
from vibe.core.types import (
AssistantEvent,
ToolCallEvent,
ToolResultEvent,
UserMessageEvent,
)
TODO_TOOL_CALL = [
{
"id": "call_todo_1",
"function": {"name": "todo", "arguments": '{"action": "read"}'},
"index": 0,
}
]
def _chunked_completion(reasoning: str, answer: str) -> dict[str, Any]:
completion = mistral_completion(answer)
completion["choices"][0]["message"]["content"] = [
{"type": "thinking", "thinking": [{"type": "text", "text": reasoning}]},
{"type": "text", "text": answer},
]
return completion
@pytest.mark.asyncio
async def test_act_completes_against_real_backend(mistral_api: MistralAPI) -> None:
# A plain prompt yields a user event then the backend's assistant reply, and
# the loop accumulates prompt+completion tokens from the usage block.
mistral_api.reply(
mistral_completion("Hi there!", prompt_tokens=120, completion_tokens=30)
)
agent = build_e2e_agent_loop()
events = [event async for event in agent.act("Hello")]
assert [type(e) for e in events] == [UserMessageEvent, AssistantEvent]
assert isinstance(events[1], AssistantEvent)
assert events[1].content == "Hi there!"
assert agent.stats.context_tokens == 150
@pytest.mark.asyncio
async def test_act_streaming(mistral_api: MistralAPI) -> None:
# Streamed SSE chunks are reassembled into the final assistant content.
_, chunks, _ = STREAMED_SIMPLE_CONVERSATION_PARAMS[0]
mistral_api.reply_stream(chunks)
agent = build_e2e_agent_loop(enable_streaming=True)
events = [event async for event in agent.act("Hi")]
assert assistant_text(events).endswith("Some content")
@pytest.mark.asyncio
async def test_act_tool_call_round_trip(mistral_api: MistralAPI) -> None:
# A tool call is parsed, executed, and its result fed back for a final reply.
mistral_api.reply(
mistral_completion("", tool_calls=TODO_TOOL_CALL),
mistral_completion("All done"),
)
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
events = [event async for event in agent.act("Show my todos")]
assert any(isinstance(e, ToolCallEvent) for e in events)
assert any(isinstance(e, ToolResultEvent) for e in events)
final = [e for e in events if isinstance(e, AssistantEvent)]
assert final and final[-1].content == "All done"
@pytest.mark.asyncio
async def test_act_serializes_tools_in_request_payload(mistral_api: MistralAPI) -> None:
# Enabled tools are serialized into the outgoing chat-completions request.
mistral_api.reply(mistral_completion("ok"))
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
_ = [event async for event in agent.act("Hello")]
names = {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
assert "todo" in names
@pytest.mark.asyncio
async def test_act_extracts_text_from_chunked_content_array(
mistral_api: MistralAPI,
) -> None:
# A thinking+text content array has its text part extracted as the reply.
mistral_api.reply(_chunked_completion("thinking hard", "the answer"))
agent = build_e2e_agent_loop()
events = [event async for event in agent.act("Why?")]
assistant = [e for e in events if isinstance(e, AssistantEvent)]
assert assistant and assistant[-1].content == "the answer"

View file

@ -0,0 +1,170 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, cast
from pydantic import BaseModel
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.tools.builtins.bash import BashResult
from vibe.core.tools.permissions import RequiredPermission
from vibe.core.types import ApprovalResponse, BaseEvent, ToolResultEvent
def _bash_call(command: str, timeout: int | None = None) -> dict[str, Any]:
arguments: dict[str, Any] = {"command": command}
if timeout is not None:
arguments["timeout"] = timeout
return mistral_completion(
"",
tool_calls=[
{
"id": "call_bash",
"function": {"name": "bash", "arguments": json.dumps(arguments)},
"index": 0,
}
],
)
async def _run_bash(
mistral_api: MistralAPI,
command: str,
*,
timeout: int | None = None,
agent_name: str = BuiltinAgentName.AUTO_APPROVE,
approval: ApprovalResponse | None = None,
) -> ToolResultEvent:
mistral_api.reply(_bash_call(command, timeout), mistral_completion("done"))
agent = build_e2e_agent_loop(
config=e2e_config(enabled_tools=["bash"]), agent_name=agent_name
)
if approval is not None:
async def approval_callback(
_tool_name: str,
_args: BaseModel,
_tool_call_id: str,
_rp: list[RequiredPermission] | None = None,
) -> tuple[ApprovalResponse, str | None]:
return (approval, None)
agent.set_approval_callback(approval_callback)
events: list[BaseEvent] = [event async for event in agent.act("go")]
return next(e for e in events if isinstance(e, ToolResultEvent))
@pytest.mark.asyncio
async def test_bash_captures_stdout(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "echo hello")
bash_result = cast(BashResult, result.result)
assert bash_result.returncode == 0
assert "hello" in bash_result.stdout
@pytest.mark.asyncio
async def test_bash_captures_stderr(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "echo oops >&2")
bash_result = cast(BashResult, result.result)
assert "oops" in bash_result.stderr
@pytest.mark.asyncio
async def test_bash_nonzero_exit_surfaces_as_error(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "exit 3")
assert result.error is not None
assert "Return code: 3" in result.error
@pytest.mark.asyncio
async def test_bash_timeout_surfaces_as_error(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "sleep 5", timeout=1)
assert result.error is not None
assert "timed out" in result.error.lower()
@pytest.mark.asyncio
async def test_bash_output_truncated_to_max_bytes(mistral_api: MistralAPI) -> None:
result = await _run_bash(mistral_api, "yes x | head -c 100000")
bash_result = cast(BashResult, result.result)
assert len(bash_result.stdout) <= 16_000
@pytest.mark.asyncio
async def test_bash_denylisted_command_is_skipped(mistral_api: MistralAPI) -> None:
result = await _run_bash(
mistral_api, "vim file.txt", agent_name=BuiltinAgentName.DEFAULT
)
assert result.skipped is True
assert result.skip_reason is not None
assert "denied" in result.skip_reason.lower()
@pytest.mark.asyncio
async def test_bash_allowlisted_command_runs_without_approval(
mistral_api: MistralAPI,
) -> None:
# No approval callback registered; an allowlisted command must run anyway.
result = await _run_bash(
mistral_api, "echo allowed", agent_name=BuiltinAgentName.DEFAULT
)
assert result.skipped is False
assert "allowed" in cast(BashResult, result.result).stdout
@pytest.mark.asyncio
async def test_bash_non_allowlisted_command_requires_approval(
mistral_api: MistralAPI,
) -> None:
result = await _run_bash(
mistral_api,
"touch newfile.txt",
agent_name=BuiltinAgentName.DEFAULT,
approval=ApprovalResponse.YES,
)
assert result.skipped is False
assert (Path.cwd() / "newfile.txt").exists()
@pytest.mark.asyncio
async def test_bash_non_allowlisted_command_denied_at_prompt_is_skipped(
mistral_api: MistralAPI,
) -> None:
result = await _run_bash(
mistral_api,
"touch denied.txt",
agent_name=BuiltinAgentName.DEFAULT,
approval=ApprovalResponse.NO,
)
assert result.skipped is True
assert not (Path.cwd() / "denied.txt").exists()
@pytest.mark.asyncio
async def test_bash_command_touching_outside_workdir_requires_approval(
mistral_api: MistralAPI, tmp_path: Path
) -> None:
outside = tmp_path / "outside.txt"
result = await _run_bash(
mistral_api,
f"touch {outside}",
agent_name=BuiltinAgentName.DEFAULT,
approval=ApprovalResponse.NO,
)
assert result.skipped is True
assert not outside.exists()

View file

@ -0,0 +1,136 @@
from __future__ import annotations
from typing import Any
import httpx
import pytest
import respx
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from tests.constants import CONNECTORS_BOOTSTRAP_PATH
from vibe.core.config import ConnectorConfig
def _connector(
*,
connector_id: str = "conn-1",
name: str = "wiki",
is_ready: bool = True,
tools: list[dict[str, Any]] | None = None,
auth_action: dict[str, Any] | None = None,
) -> dict[str, Any]:
return {
"id": connector_id,
"name": name,
"status": {"is_ready": is_ready},
"tools": tools or [],
"auth_action": auth_action,
}
def _tool(name: str = "search", description: str = "Search docs") -> dict[str, Any]:
return {
"name": name,
"description": description,
"inputSchema": {"type": "object", "properties": {"query": {"type": "string"}}},
}
def _set_bootstrap(router: respx.MockRouter, connectors: list[dict[str, Any]]) -> None:
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
return_value=httpx.Response(200, json={"connectors": connectors})
)
def _connectors_config(*aliases: str) -> Any:
return e2e_config(
connectors=[ConnectorConfig(name=alias, disabled=False) for alias in aliases]
)
async def _offered_tool_names(
mistral_api: MistralAPI, *enabled_connectors: str
) -> set[str]:
# Drive one turn and read which tools the AgentLoop serialized into the
# outgoing chat-completions request — the only connector-visible surface.
mistral_api.reply(mistral_completion("ok"))
agent = build_e2e_agent_loop(config=_connectors_config(*enabled_connectors))
_ = [event async for event in agent.act("hello")]
return {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
@pytest.mark.asyncio
async def test_connector_tools_are_offered_to_the_model(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(
mock_mistral, [_connector(name="wiki", tools=[_tool("search"), _tool("read")])]
)
names = await _offered_tool_names(mistral_api, "wiki")
assert "connector_wiki_search" in names
assert "connector_wiki_read" in names
@pytest.mark.asyncio
async def test_colliding_connector_aliases_are_disambiguated(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(
mock_mistral,
[
_connector(connector_id="c-1", name="mcp", tools=[_tool("a")]),
_connector(connector_id="c-2", name="mcp", tools=[_tool("b")]),
],
)
names = await _offered_tool_names(mistral_api, "mcp", "mcp_2")
assert "connector_mcp_a" in names
assert "connector_mcp_2_b" in names
@pytest.mark.asyncio
async def test_not_ready_connector_offers_no_tools(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(
mock_mistral,
[
_connector(
name="linear",
is_ready=False,
tools=[_tool("search")],
auth_action={"type": "oauth"},
)
],
)
names = await _offered_tool_names(mistral_api, "linear")
assert not any(name.startswith("connector_linear") for name in names)
@pytest.mark.asyncio
async def test_disabled_connector_tools_are_withheld(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
_set_bootstrap(mock_mistral, [_connector(name="wiki", tools=[_tool("search")])])
# No ConnectorConfig entry → the connector stays disabled by default.
names = await _offered_tool_names(mistral_api)
assert "connector_wiki_search" not in names
@pytest.mark.asyncio
async def test_bootstrap_failure_still_lets_the_agent_run(
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
) -> None:
mock_mistral.get(CONNECTORS_BOOTSTRAP_PATH).mock(return_value=httpx.Response(500))
names = await _offered_tool_names(mistral_api, "wiki")
assert not any(name.startswith("connector_") for name in names)

View file

@ -0,0 +1,144 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.tools.builtins.ask_user_question import (
AskUserQuestionArgs,
AskUserQuestionResult,
)
from vibe.core.types import PlanReviewEndedEvent, PlanReviewRequestedEvent, Role
from vibe.core.utils.tags import VIBE_WARNING_TAG
EXIT_PLAN_TOOL_CALL = [
{
"id": "call_exit_plan_1",
"function": {"name": "exit_plan_mode", "arguments": "{}"},
"index": 0,
}
]
def _user_message_ids(agent: Any) -> list[str]:
return [m.message_id for m in agent.messages if m.role == Role.user]
def _assistant_message_id(agent: Any) -> str:
return next(m.message_id for m in agent.messages if m.role == Role.assistant)
@pytest.mark.asyncio
async def test_fork_copies_all_non_system_messages(mistral_api: MistralAPI) -> None:
# fork() with no anchor clones every non-system message into the child loop.
mistral_api.reply(mistral_completion("Hi there!"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Hello")]
forked = await agent.fork()
non_system = [m for m in forked.messages if m.role != Role.system]
assert [m.role for m in non_system] == [Role.user, Role.assistant]
assert forked.parent_session_id == agent.session_id
assert forked.session_id != agent.session_id
@pytest.mark.asyncio
async def test_fork_from_message_id_truncates_at_next_user_turn(
mistral_api: MistralAPI,
) -> None:
# Forking from a user message keeps that turn but drops everything from the next one.
mistral_api.reply(mistral_completion("First"), mistral_completion("Second"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Turn one")]
_ = [event async for event in agent.act("Turn two")]
first_user_id = _user_message_ids(agent)[0]
forked = await agent.fork(first_user_id)
contents = [m.content for m in forked.messages if m.role != Role.system]
assert contents == ["Turn one", "First"]
@pytest.mark.asyncio
async def test_fork_from_unknown_message_id_raises(mistral_api: MistralAPI) -> None:
# An unknown anchor id is rejected rather than silently forking everything.
mistral_api.reply(mistral_completion("Hi"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Hello")]
with pytest.raises(ValueError, match="unknown message_id"):
await agent.fork("does-not-exist")
@pytest.mark.asyncio
async def test_fork_from_assistant_message_id_raises(mistral_api: MistralAPI) -> None:
# Forking is only allowed from user turns; an assistant anchor is rejected.
mistral_api.reply(mistral_completion("Hi"))
agent = build_e2e_agent_loop()
_ = [event async for event in agent.act("Hello")]
assistant_id = _assistant_message_id(agent)
with pytest.raises(ValueError, match="only supported for user messages"):
await agent.fork(assistant_id)
def _plan_agent(mistral_api: MistralAPI) -> Any:
mistral_api.reply(
mistral_completion("", tool_calls=EXIT_PLAN_TOOL_CALL),
mistral_completion("Staying in plan mode."),
)
return build_e2e_agent_loop(
config=e2e_config(enabled_tools=["exit_plan_mode"]),
agent_name=BuiltinAgentName.PLAN,
)
@pytest.mark.asyncio
async def test_plan_mode_emits_review_requested_and_ended_events(
mistral_api: MistralAPI,
) -> None:
# An exit_plan_mode round trip brackets the tool with review-requested/ended events.
agent = _plan_agent(mistral_api)
async def stay_in_plan(_: AskUserQuestionArgs) -> AskUserQuestionResult:
return AskUserQuestionResult(cancelled=True, answers=[])
agent.set_user_input_callback(stay_in_plan)
events = [event async for event in agent.act("Make a plan")]
assert any(isinstance(e, PlanReviewRequestedEvent) for e in events)
assert any(isinstance(e, PlanReviewEndedEvent) for e in events)
@pytest.mark.asyncio
async def test_plan_mode_injects_updated_plan_when_file_changed(
mistral_api: MistralAPI,
) -> None:
# If the plan file changes during review, its new content is injected back as context.
agent = _plan_agent(mistral_api)
plan_path: Path | None = None
async def edit_plan_then_decline(_: AskUserQuestionArgs) -> AskUserQuestionResult:
assert plan_path is not None
plan_path.parent.mkdir(parents=True, exist_ok=True)
plan_path.write_text("# Updated plan\nStep 1")
return AskUserQuestionResult(cancelled=True, answers=[])
agent.set_user_input_callback(edit_plan_then_decline)
async for event in agent.act("Make a plan"):
if isinstance(event, PlanReviewRequestedEvent):
plan_path = event.file_path
injected = [m for m in agent.messages if getattr(m, "injected", False)]
assert any(
m.content and VIBE_WARNING_TAG in m.content and "# Updated plan" in m.content
for m in injected
)

View file

@ -0,0 +1,178 @@
from __future__ import annotations
import pytest
from tests.agent_loop.e2e.conftest import (
ProviderAPI,
anthropic_e2e_config,
assistant_text,
build_e2e_agent_loop,
openai_responses_e2e_config,
)
from tests.backend.data.anthropic import (
anthropic_message,
anthropic_reasoning_tool_use_stream,
anthropic_request_content_blocks,
anthropic_text_stream,
anthropic_tool_use,
)
from tests.backend.data.openai_responses import (
openai_function_call_item,
openai_message_item,
openai_reasoning_tool_call_stream,
openai_response,
openai_text_stream,
)
from vibe.core.types import ToolResultEvent
class TestAnthropic:
@pytest.mark.asyncio
async def test_agent_answers(self, anthropic_api: ProviderAPI) -> None:
# A plain prompt is answered through the Anthropic messages wire.
anthropic_api.reply(anthropic_message("pong"))
agent = build_e2e_agent_loop(config=anthropic_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 15
@pytest.mark.asyncio
async def test_agent_streams(self, anthropic_api: ProviderAPI) -> None:
# Anthropic SSE deltas are reassembled into the final assistant content.
anthropic_api.reply_stream(anthropic_text_stream("pong"))
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(), enable_streaming=True
)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_executes_tool_call(self, anthropic_api: ProviderAPI) -> None:
# A tool_use turn runs the tool, then Anthropic returns the final answer.
anthropic_api.reply(
anthropic_tool_use("todo", {"action": "read"}),
anthropic_message("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"])
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, anthropic_api: ProviderAPI
) -> None:
# A streamed thinking + tool_use turn runs the tool, then replays that
# reasoning/tool history into the follow-up Anthropic request.
anthropic_api.reply_streams(
anthropic_reasoning_tool_use_stream("todo", '{"action": "read"}'),
anthropic_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"]), enable_streaming=True
)
events = [event async for event in agent.act("What's on my todo list?")]
assert "Your list is empty." in assistant_text(events)
blocks = anthropic_request_content_blocks(anthropic_api.request_json)
thinking = [b["thinking"] for b in blocks if b.get("type") == "thinking"]
tool_uses = [b for b in blocks if b.get("type") == "tool_use"]
assert any("thinking..." in text for text in thinking)
assert tool_uses
class TestOpenAIResponses:
@pytest.mark.asyncio
async def test_agent_answers(self, openai_responses_api: ProviderAPI) -> None:
# A plain prompt is answered through the OpenAI Responses wire.
openai_responses_api.reply(openai_response([openai_message_item("pong")]))
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 12
@pytest.mark.asyncio
async def test_agent_streams(self, openai_responses_api: ProviderAPI) -> None:
# OpenAI Responses SSE deltas are reassembled into the final content.
openai_responses_api.reply_stream(openai_text_stream("pong"))
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(), enable_streaming=True
)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_captures_reasoning(
self, openai_responses_api: ProviderAPI
) -> None:
# Commentary phase output is captured as reasoning on the assistant message.
openai_responses_api.reply(
openai_response(
[
openai_message_item("Let me think.", phase="commentary"),
openai_message_item("pong", phase="final_answer"),
],
output_tokens=5,
)
)
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert any(
m.reasoning_content and "Let me think." in m.reasoning_content
for m in agent.messages
)
@pytest.mark.asyncio
async def test_agent_executes_tool_call(
self, openai_responses_api: ProviderAPI
) -> None:
# A function_call item runs the tool, then OpenAI returns the final answer.
openai_responses_api.reply(
openai_response([openai_function_call_item("todo", '{"action": "read"}')]),
openai_response([openai_message_item("Your list is empty.")]),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"])
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, openai_responses_api: ProviderAPI
) -> None:
# A streamed commentary + function_call turn runs the tool, then OpenAI
# streams the final answer on the follow-up request.
openai_responses_api.reply_streams(
openai_reasoning_tool_call_stream("todo", '{"action": "read"}'),
openai_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"]),
enable_streaming=True,
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)

View file

@ -0,0 +1,224 @@
from __future__ import annotations
from collections.abc import Iterator
from pathlib import Path
from git import Repo
import httpx
import pytest
import respx
from tests.agent_loop.e2e.conftest import build_e2e_agent_loop
from tests.constants import (
CONNECTORS_BOOTSTRAP_PATH,
MISTRAL_BASE_URL,
TELEPORT_COMPLETE_URL,
TELEPORT_SESSIONS_PATH,
)
from vibe.core.agent_loop import AgentLoop, TeleportError
from vibe.core.teleport.types import (
TeleportCheckingGitEvent,
TeleportCompleteEvent,
TeleportPushingEvent,
TeleportPushRequiredEvent,
TeleportPushResponseEvent,
TeleportStartingWorkflowEvent,
)
# vibe_code_sessions_base_url is an exclude=True field, so it is dropped when the
# agent manager re-derives config via model_dump(); the default url is unavoidable.
SESSIONS_BASE_URL = "https://chat.mistral.ai"
SESSIONS_URL = f"{SESSIONS_BASE_URL}{TELEPORT_SESSIONS_PATH}"
def _sessions_ok() -> httpx.Response:
return httpx.Response(
200,
json={
"sessionId": "controller-session-id",
"webSessionId": "web-session-id",
"projectId": "project-id",
"status": "running",
"url": TELEPORT_COMPLETE_URL,
},
)
def _commit(repo: Repo, message: str) -> str:
(Path(repo.working_dir) / "file.txt").write_text(f"{message}\n")
repo.index.add(["file.txt"])
repo.index.commit(message)
return repo.head.commit.hexsha
def _init_repo(workdir: Path) -> Repo:
# origin is a local bare repo so fetch/push stay offline and instant; a
# separate github-url remote satisfies Teleport's GitHub-only detection.
bare = Repo.init(workdir.with_name(f"{workdir.name}_origin.git"), bare=True)
repo = Repo.init(workdir, initial_branch="work")
repo.config_writer().set_value("user", "name", "Tester").release()
repo.config_writer().set_value("user", "email", "t@example.com").release()
repo.create_remote("origin", str(bare.git_dir))
repo.create_remote("hub", "https://github.com/owner/repo.git")
return repo
def _repo_with_pushed_branch(workdir: Path) -> Repo:
repo = _init_repo(workdir)
_commit(repo, "initial")
repo.git.push("origin", "work")
return repo
@pytest.fixture
def mock_sessions() -> Iterator[respx.MockRouter]:
# One router stubs both hosts: the connector bootstrap (api.mistral.ai, hit on
# agent init) and the teleport sessions endpoint (chat.mistral.ai).
with respx.mock(assert_all_called=False) as router:
router.get(f"{MISTRAL_BASE_URL}{CONNECTORS_BOOTSTRAP_PATH}").mock(
return_value=httpx.Response(200, json={"connectors": []})
)
router.post(SESSIONS_URL).mock(return_value=_sessions_ok())
yield router
async def _drain(
agent: AgentLoop, prompt: str | None, *, approve: bool | None = None
) -> list[object]:
gen = agent.teleport_to_vibe_code(prompt)
events: list[object] = []
response: TeleportPushResponseEvent | None = None
while True:
try:
event = await gen.asend(response)
except StopAsyncIteration:
break
events.append(event)
response = None
if isinstance(event, TeleportPushRequiredEvent) and approve is not None:
response = TeleportPushResponseEvent(approved=approve)
return events
@pytest.mark.asyncio
async def test_teleport_completes_when_branch_already_pushed(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
_repo_with_pushed_branch(tmp_working_directory)
events = await _drain(build_e2e_agent_loop(), "do the thing")
assert [type(e) for e in events] == [
TeleportCheckingGitEvent,
TeleportStartingWorkflowEvent,
TeleportCompleteEvent,
]
assert isinstance(events[-1], TeleportCompleteEvent)
assert events[-1].url == TELEPORT_COMPLETE_URL
@pytest.mark.asyncio
async def test_teleport_sends_repo_metadata_and_diff(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
commit = repo.head.commit.hexsha
(tmp_working_directory / "file.txt").write_text("uncommitted change\n")
await _drain(build_e2e_agent_loop(), "ship it")
payload = mock_sessions.calls.last.request.read().decode()
assert "https://github.com/owner/repo.git" in payload
assert "work" in payload
assert commit in payload
assert "zstd" in payload
@pytest.mark.asyncio
async def test_teleport_pushes_then_completes_when_approved(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
head = _commit(repo, "second")
events = await _drain(build_e2e_agent_loop(), "ship it", approve=True)
assert [type(e) for e in events] == [
TeleportCheckingGitEvent,
TeleportPushRequiredEvent,
TeleportPushingEvent,
TeleportStartingWorkflowEvent,
TeleportCompleteEvent,
]
assert repo.remote("origin").refs["work"].commit.hexsha == head
@pytest.mark.asyncio
async def test_teleport_aborts_when_push_declined(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
_commit(repo, "second")
with pytest.raises(TeleportError, match="not pushed"):
await _drain(build_e2e_agent_loop(), "ship it", approve=False)
assert not mock_sessions.post(SESSIONS_URL).called
@pytest.mark.asyncio
async def test_teleport_fails_when_push_fails(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
_commit(repo, "second")
repo.git.remote("set-url", "--push", "origin", "/nonexistent/repo.git")
with pytest.raises(TeleportError, match="Failed to push"):
await _drain(build_e2e_agent_loop(), "ship it", approve=True)
@pytest.mark.asyncio
async def test_teleport_requires_a_branch(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _repo_with_pushed_branch(tmp_working_directory)
repo.git.checkout(repo.head.commit.hexsha) # detach HEAD
with pytest.raises(TeleportError, match="checked-out branch"):
await _drain(build_e2e_agent_loop(), "ship it")
@pytest.mark.asyncio
async def test_teleport_rejects_empty_prompt(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = _init_repo(tmp_working_directory)
_commit(repo, "initial")
with pytest.raises(TeleportError, match="non-empty prompt"):
await _drain(build_e2e_agent_loop(), None)
@pytest.mark.asyncio
async def test_teleport_surfaces_http_error(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
_repo_with_pushed_branch(tmp_working_directory)
mock_sessions.post(SESSIONS_URL).mock(return_value=httpx.Response(500, text="boom"))
with pytest.raises(TeleportError, match="start failed"):
await _drain(build_e2e_agent_loop(), "ship it")
@pytest.mark.asyncio
async def test_teleport_unsupported_without_github_remote(
tmp_working_directory: Path, mock_sessions: respx.MockRouter
) -> None:
repo = Repo.init(tmp_working_directory, initial_branch="work")
repo.config_writer().set_value("user", "name", "Tester").release()
repo.config_writer().set_value("user", "email", "t@example.com").release()
_commit(repo, "initial")
with pytest.raises(TeleportError, match="GitHub"):
await _drain(build_e2e_agent_loop(), "ship it")

View file

@ -0,0 +1,93 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, cast
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.backend.data.mistral import mistral_completion
from vibe.core.tools.builtins.grep import GrepResult
from vibe.core.tools.builtins.read import ReadResult
from vibe.core.tools.builtins.todo import TodoResult
from vibe.core.types import BaseEvent, ToolResultEvent
def _tool_call(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
return mistral_completion(
"",
tool_calls=[
{
"id": f"call_{name}",
"function": {"name": name, "arguments": json.dumps(arguments)},
"index": 0,
}
],
)
async def _run_tool(
mistral_api: MistralAPI, name: str, arguments: dict[str, Any]
) -> ToolResultEvent:
# Drive one tool call against the real cwd, then a final reply, and return
# the single tool result for the test to assert on.
mistral_api.reply(_tool_call(name, arguments), mistral_completion("done"))
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=[name]))
events: list[BaseEvent] = [event async for event in agent.act("go")]
result = next(e for e in events if isinstance(e, ToolResultEvent))
assert result.error is None
return result
@pytest.mark.asyncio
async def test_write_file_tool_creates_file(mistral_api: MistralAPI) -> None:
target = Path.cwd() / "note.txt"
await _run_tool(mistral_api, "write_file", {"path": str(target), "content": "hi\n"})
assert target.read_text() == "hi\n"
@pytest.mark.asyncio
async def test_read_tool_returns_file_content(mistral_api: MistralAPI) -> None:
target = Path.cwd() / "note.txt"
target.write_text("hello world\n")
result = await _run_tool(mistral_api, "read", {"file_path": str(target)})
assert "hello world" in cast(ReadResult, result.result).content
@pytest.mark.asyncio
async def test_edit_tool_replaces_text(mistral_api: MistralAPI) -> None:
target = Path.cwd() / "note.txt"
target.write_text("hello world\n")
await _run_tool(
mistral_api,
"edit",
{"file_path": str(target), "old_string": "hello", "new_string": "goodbye"},
)
assert target.read_text() == "goodbye world\n"
@pytest.mark.asyncio
async def test_grep_tool_finds_matches(mistral_api: MistralAPI) -> None:
(Path.cwd() / "note.txt").write_text("needle here\n")
result = await _run_tool(mistral_api, "grep", {"pattern": "needle", "path": "."})
assert cast(GrepResult, result.result).match_count >= 1
@pytest.mark.asyncio
async def test_todo_tool_writes_items(mistral_api: MistralAPI) -> None:
result = await _run_tool(
mistral_api,
"todo",
{"action": "write", "todos": [{"id": "1", "content": "ship it"}]},
)
assert cast(TodoResult, result.result).total_count == 1