v2.17.0 (#822)
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai> Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com> Co-authored-by: Hdandria <henri.dandria@mistral.ai> Co-authored-by: Ivana Dunisijevic <ivana.dunisijevic@mistral.ai> Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com> Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai> Co-authored-by: Mert Unsal <mert.unsal@mistral.ai> Co-authored-by: Michel Thomazo <51709227+michelTho@users.noreply.github.com> Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com> Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai> Co-authored-by: Val <102326092+vdeva@users.noreply.github.com> Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com> Co-authored-by: renovate-mistral[bot] <253709520+renovate-mistral[bot]@users.noreply.github.com> Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
parent
564a14365e
commit
6bedf271ce
223 changed files with 10533 additions and 6947 deletions
0
tests/agent_loop/e2e/__init__.py
Normal file
0
tests/agent_loop/e2e/__init__.py
Normal file
181
tests/agent_loop/e2e/conftest.py
Normal file
181
tests/agent_loop/e2e/conftest.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from tests.conftest import build_test_agent_loop, build_test_vibe_config
|
||||
from tests.constants import (
|
||||
ANTHROPIC_BASE_URL,
|
||||
ANTHROPIC_MESSAGES_PATH,
|
||||
CHAT_COMPLETIONS_PATH,
|
||||
CONNECTORS_BOOTSTRAP_PATH,
|
||||
MISTRAL_BASE_URL,
|
||||
OPENAI_BASE_URL,
|
||||
OPENAI_RESPONSES_PATH,
|
||||
)
|
||||
from vibe.core.agent_loop import AgentLoop
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
|
||||
from vibe.core.llm.backend.factory import BACKEND_FACTORY
|
||||
from vibe.core.types import AssistantEvent, Backend, BaseEvent
|
||||
|
||||
|
||||
def assistant_text(events: list[BaseEvent]) -> str:
|
||||
return "".join(
|
||||
e.content for e in events if isinstance(e, AssistantEvent) and e.content
|
||||
).strip()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _git_offline(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
# Restrict git to the local file protocol so no test can fetch/push over the
|
||||
# network, even if a remote is misconfigured to a real URL.
|
||||
monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file")
|
||||
|
||||
|
||||
def e2e_config(**overrides: Any) -> VibeConfig:
|
||||
# Defaults give the mistral provider/model (api.mistral.ai, the respx-mocked
|
||||
# base). The default model reasons, but that only adds reasoning_effort to the
|
||||
# request, which the mocked responses ignore.
|
||||
# The e2e harness mocks the connector bootstrap endpoint via respx, so
|
||||
# connector discovery is enabled (and fast) here unlike the unit-test default.
|
||||
return build_test_vibe_config(
|
||||
enabled_tools=overrides.pop("enabled_tools", []),
|
||||
enable_connectors=overrides.pop("enable_connectors", True),
|
||||
system_prompt_id="tests",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
def _generic_e2e_config(
|
||||
*, name: str, api_base: str, api_style: str, model: str, **overrides: Any
|
||||
) -> VibeConfig:
|
||||
provider = ProviderConfig(
|
||||
name=name,
|
||||
api_base=api_base,
|
||||
api_key_env_var=f"{name.upper()}_API_KEY",
|
||||
api_style=api_style,
|
||||
backend=Backend.GENERIC,
|
||||
)
|
||||
models = [ModelConfig(name=model, provider=name, alias=name)]
|
||||
return e2e_config(
|
||||
active_model=name, models=models, providers=[provider], **overrides
|
||||
)
|
||||
|
||||
|
||||
def anthropic_e2e_config(**overrides: Any) -> VibeConfig:
|
||||
return _generic_e2e_config(
|
||||
name="anthropic",
|
||||
api_base=ANTHROPIC_BASE_URL,
|
||||
api_style="anthropic",
|
||||
model="claude-test",
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
def openai_responses_e2e_config(**overrides: Any) -> VibeConfig:
|
||||
return _generic_e2e_config(
|
||||
name="openai",
|
||||
api_base=f"{OPENAI_BASE_URL}/v1",
|
||||
api_style="openai-responses",
|
||||
model="gpt-test",
|
||||
**overrides,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mistral() -> Iterator[respx.MockRouter]:
|
||||
with respx.mock(base_url=MISTRAL_BASE_URL, assert_all_called=False) as router:
|
||||
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
|
||||
return_value=httpx.Response(200, json={"connectors": []})
|
||||
)
|
||||
yield router
|
||||
|
||||
|
||||
def build_e2e_agent_loop(
|
||||
*, config: VibeConfig | None = None, enable_streaming: bool = False, **kwargs: Any
|
||||
) -> AgentLoop:
|
||||
resolved_config = config or e2e_config()
|
||||
provider = resolved_config.providers[0]
|
||||
# todo now we use build_test_agent_loop for the fake mcp registry.
|
||||
# Maybe mock http tool calls and instantiate a real registry later for more coverage
|
||||
return build_test_agent_loop(
|
||||
config=resolved_config,
|
||||
agent_name=kwargs.pop("agent_name", BuiltinAgentName.AUTO_APPROVE),
|
||||
backend=BACKEND_FACTORY[provider.backend](provider=provider),
|
||||
enable_streaming=enable_streaming,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class ProviderAPI:
|
||||
"""Stubs a provider's completion wire; the AgentLoop stays the subject."""
|
||||
|
||||
def __init__(self, router: respx.MockRouter, path: str) -> None:
|
||||
self.route = router.post(path)
|
||||
|
||||
def reply(self, *completions: dict[str, Any]) -> None:
|
||||
responses = [httpx.Response(200, json=c) for c in completions]
|
||||
if len(responses) == 1:
|
||||
self.route.mock(return_value=responses[0])
|
||||
else:
|
||||
self.route.mock(side_effect=responses)
|
||||
|
||||
def reply_stream(self, chunks: list[bytes]) -> None:
|
||||
self.route.mock(return_value=self._stream_response(chunks))
|
||||
|
||||
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
|
||||
self.route.mock(
|
||||
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stream_response(chunks: list[bytes]) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
|
||||
headers={"Content-Type": "text/event-stream"},
|
||||
)
|
||||
|
||||
@property
|
||||
def request_json(self) -> dict[str, Any]:
|
||||
return json.loads(self.route.calls.last.request.content)
|
||||
|
||||
|
||||
class MistralAPI(ProviderAPI):
|
||||
def __init__(self, router: respx.MockRouter) -> None:
|
||||
super().__init__(router, CHAT_COMPLETIONS_PATH)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mistral_api(mock_mistral: respx.MockRouter) -> MistralAPI:
|
||||
return MistralAPI(mock_mistral)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_anthropic() -> Iterator[respx.MockRouter]:
|
||||
with respx.mock(base_url=ANTHROPIC_BASE_URL, assert_all_called=False) as router:
|
||||
yield router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def anthropic_api(mock_anthropic: respx.MockRouter) -> ProviderAPI:
|
||||
return ProviderAPI(mock_anthropic, ANTHROPIC_MESSAGES_PATH)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_openai() -> Iterator[respx.MockRouter]:
|
||||
with respx.mock(base_url=OPENAI_BASE_URL, assert_all_called=False) as router:
|
||||
yield router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def openai_responses_api(mock_openai: respx.MockRouter) -> ProviderAPI:
|
||||
return ProviderAPI(mock_openai, OPENAI_RESPONSES_PATH)
|
||||
111
tests/agent_loop/e2e/test_e2e_agent_loop.py
Normal file
111
tests/agent_loop/e2e/test_e2e_agent_loop.py
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import (
|
||||
MistralAPI,
|
||||
assistant_text,
|
||||
build_e2e_agent_loop,
|
||||
e2e_config,
|
||||
)
|
||||
from tests.backend.data.mistral import (
|
||||
STREAMED_SIMPLE_CONVERSATION_PARAMS,
|
||||
mistral_completion,
|
||||
)
|
||||
from vibe.core.types import (
|
||||
AssistantEvent,
|
||||
ToolCallEvent,
|
||||
ToolResultEvent,
|
||||
UserMessageEvent,
|
||||
)
|
||||
|
||||
TODO_TOOL_CALL = [
|
||||
{
|
||||
"id": "call_todo_1",
|
||||
"function": {"name": "todo", "arguments": '{"action": "read"}'},
|
||||
"index": 0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _chunked_completion(reasoning: str, answer: str) -> dict[str, Any]:
|
||||
completion = mistral_completion(answer)
|
||||
completion["choices"][0]["message"]["content"] = [
|
||||
{"type": "thinking", "thinking": [{"type": "text", "text": reasoning}]},
|
||||
{"type": "text", "text": answer},
|
||||
]
|
||||
return completion
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_completes_against_real_backend(mistral_api: MistralAPI) -> None:
|
||||
# A plain prompt yields a user event then the backend's assistant reply, and
|
||||
# the loop accumulates prompt+completion tokens from the usage block.
|
||||
mistral_api.reply(
|
||||
mistral_completion("Hi there!", prompt_tokens=120, completion_tokens=30)
|
||||
)
|
||||
agent = build_e2e_agent_loop()
|
||||
|
||||
events = [event async for event in agent.act("Hello")]
|
||||
|
||||
assert [type(e) for e in events] == [UserMessageEvent, AssistantEvent]
|
||||
assert isinstance(events[1], AssistantEvent)
|
||||
assert events[1].content == "Hi there!"
|
||||
assert agent.stats.context_tokens == 150
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_streaming(mistral_api: MistralAPI) -> None:
|
||||
# Streamed SSE chunks are reassembled into the final assistant content.
|
||||
_, chunks, _ = STREAMED_SIMPLE_CONVERSATION_PARAMS[0]
|
||||
mistral_api.reply_stream(chunks)
|
||||
agent = build_e2e_agent_loop(enable_streaming=True)
|
||||
|
||||
events = [event async for event in agent.act("Hi")]
|
||||
|
||||
assert assistant_text(events).endswith("Some content")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_tool_call_round_trip(mistral_api: MistralAPI) -> None:
|
||||
# A tool call is parsed, executed, and its result fed back for a final reply.
|
||||
mistral_api.reply(
|
||||
mistral_completion("", tool_calls=TODO_TOOL_CALL),
|
||||
mistral_completion("All done"),
|
||||
)
|
||||
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
|
||||
|
||||
events = [event async for event in agent.act("Show my todos")]
|
||||
|
||||
assert any(isinstance(e, ToolCallEvent) for e in events)
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
final = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assert final and final[-1].content == "All done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_serializes_tools_in_request_payload(mistral_api: MistralAPI) -> None:
|
||||
# Enabled tools are serialized into the outgoing chat-completions request.
|
||||
mistral_api.reply(mistral_completion("ok"))
|
||||
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
|
||||
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
names = {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
|
||||
assert "todo" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_act_extracts_text_from_chunked_content_array(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# A thinking+text content array has its text part extracted as the reply.
|
||||
mistral_api.reply(_chunked_completion("thinking hard", "the answer"))
|
||||
agent = build_e2e_agent_loop()
|
||||
|
||||
events = [event async for event in agent.act("Why?")]
|
||||
|
||||
assistant = [e for e in events if isinstance(e, AssistantEvent)]
|
||||
assert assistant and assistant[-1].content == "the answer"
|
||||
170
tests/agent_loop/e2e/test_e2e_bash.py
Normal file
170
tests/agent_loop/e2e/test_e2e_bash.py
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
from pydantic import BaseModel
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.tools.builtins.bash import BashResult
|
||||
from vibe.core.tools.permissions import RequiredPermission
|
||||
from vibe.core.types import ApprovalResponse, BaseEvent, ToolResultEvent
|
||||
|
||||
|
||||
def _bash_call(command: str, timeout: int | None = None) -> dict[str, Any]:
|
||||
arguments: dict[str, Any] = {"command": command}
|
||||
if timeout is not None:
|
||||
arguments["timeout"] = timeout
|
||||
return mistral_completion(
|
||||
"",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": "call_bash",
|
||||
"function": {"name": "bash", "arguments": json.dumps(arguments)},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
async def _run_bash(
|
||||
mistral_api: MistralAPI,
|
||||
command: str,
|
||||
*,
|
||||
timeout: int | None = None,
|
||||
agent_name: str = BuiltinAgentName.AUTO_APPROVE,
|
||||
approval: ApprovalResponse | None = None,
|
||||
) -> ToolResultEvent:
|
||||
mistral_api.reply(_bash_call(command, timeout), mistral_completion("done"))
|
||||
agent = build_e2e_agent_loop(
|
||||
config=e2e_config(enabled_tools=["bash"]), agent_name=agent_name
|
||||
)
|
||||
if approval is not None:
|
||||
|
||||
async def approval_callback(
|
||||
_tool_name: str,
|
||||
_args: BaseModel,
|
||||
_tool_call_id: str,
|
||||
_rp: list[RequiredPermission] | None = None,
|
||||
) -> tuple[ApprovalResponse, str | None]:
|
||||
return (approval, None)
|
||||
|
||||
agent.set_approval_callback(approval_callback)
|
||||
|
||||
events: list[BaseEvent] = [event async for event in agent.act("go")]
|
||||
return next(e for e in events if isinstance(e, ToolResultEvent))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_captures_stdout(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "echo hello")
|
||||
|
||||
bash_result = cast(BashResult, result.result)
|
||||
assert bash_result.returncode == 0
|
||||
assert "hello" in bash_result.stdout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_captures_stderr(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "echo oops >&2")
|
||||
|
||||
bash_result = cast(BashResult, result.result)
|
||||
assert "oops" in bash_result.stderr
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_nonzero_exit_surfaces_as_error(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "exit 3")
|
||||
|
||||
assert result.error is not None
|
||||
assert "Return code: 3" in result.error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_timeout_surfaces_as_error(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "sleep 5", timeout=1)
|
||||
|
||||
assert result.error is not None
|
||||
assert "timed out" in result.error.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_output_truncated_to_max_bytes(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(mistral_api, "yes x | head -c 100000")
|
||||
|
||||
bash_result = cast(BashResult, result.result)
|
||||
assert len(bash_result.stdout) <= 16_000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_denylisted_command_is_skipped(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_bash(
|
||||
mistral_api, "vim file.txt", agent_name=BuiltinAgentName.DEFAULT
|
||||
)
|
||||
|
||||
assert result.skipped is True
|
||||
assert result.skip_reason is not None
|
||||
assert "denied" in result.skip_reason.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_allowlisted_command_runs_without_approval(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# No approval callback registered; an allowlisted command must run anyway.
|
||||
result = await _run_bash(
|
||||
mistral_api, "echo allowed", agent_name=BuiltinAgentName.DEFAULT
|
||||
)
|
||||
|
||||
assert result.skipped is False
|
||||
assert "allowed" in cast(BashResult, result.result).stdout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_non_allowlisted_command_requires_approval(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
result = await _run_bash(
|
||||
mistral_api,
|
||||
"touch newfile.txt",
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
approval=ApprovalResponse.YES,
|
||||
)
|
||||
|
||||
assert result.skipped is False
|
||||
assert (Path.cwd() / "newfile.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_non_allowlisted_command_denied_at_prompt_is_skipped(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
result = await _run_bash(
|
||||
mistral_api,
|
||||
"touch denied.txt",
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
approval=ApprovalResponse.NO,
|
||||
)
|
||||
|
||||
assert result.skipped is True
|
||||
assert not (Path.cwd() / "denied.txt").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bash_command_touching_outside_workdir_requires_approval(
|
||||
mistral_api: MistralAPI, tmp_path: Path
|
||||
) -> None:
|
||||
outside = tmp_path / "outside.txt"
|
||||
result = await _run_bash(
|
||||
mistral_api,
|
||||
f"touch {outside}",
|
||||
agent_name=BuiltinAgentName.DEFAULT,
|
||||
approval=ApprovalResponse.NO,
|
||||
)
|
||||
|
||||
assert result.skipped is True
|
||||
assert not outside.exists()
|
||||
136
tests/agent_loop/e2e/test_e2e_connectors.py
Normal file
136
tests/agent_loop/e2e/test_e2e_connectors.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from tests.constants import CONNECTORS_BOOTSTRAP_PATH
|
||||
from vibe.core.config import ConnectorConfig
|
||||
|
||||
|
||||
def _connector(
|
||||
*,
|
||||
connector_id: str = "conn-1",
|
||||
name: str = "wiki",
|
||||
is_ready: bool = True,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
auth_action: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"id": connector_id,
|
||||
"name": name,
|
||||
"status": {"is_ready": is_ready},
|
||||
"tools": tools or [],
|
||||
"auth_action": auth_action,
|
||||
}
|
||||
|
||||
|
||||
def _tool(name: str = "search", description: str = "Search docs") -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"inputSchema": {"type": "object", "properties": {"query": {"type": "string"}}},
|
||||
}
|
||||
|
||||
|
||||
def _set_bootstrap(router: respx.MockRouter, connectors: list[dict[str, Any]]) -> None:
|
||||
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
|
||||
return_value=httpx.Response(200, json={"connectors": connectors})
|
||||
)
|
||||
|
||||
|
||||
def _connectors_config(*aliases: str) -> Any:
|
||||
return e2e_config(
|
||||
connectors=[ConnectorConfig(name=alias, disabled=False) for alias in aliases]
|
||||
)
|
||||
|
||||
|
||||
async def _offered_tool_names(
|
||||
mistral_api: MistralAPI, *enabled_connectors: str
|
||||
) -> set[str]:
|
||||
# Drive one turn and read which tools the AgentLoop serialized into the
|
||||
# outgoing chat-completions request — the only connector-visible surface.
|
||||
mistral_api.reply(mistral_completion("ok"))
|
||||
agent = build_e2e_agent_loop(config=_connectors_config(*enabled_connectors))
|
||||
_ = [event async for event in agent.act("hello")]
|
||||
return {t["function"]["name"] for t in mistral_api.request_json.get("tools", [])}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connector_tools_are_offered_to_the_model(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(
|
||||
mock_mistral, [_connector(name="wiki", tools=[_tool("search"), _tool("read")])]
|
||||
)
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "wiki")
|
||||
|
||||
assert "connector_wiki_search" in names
|
||||
assert "connector_wiki_read" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_connector_aliases_are_disambiguated(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(
|
||||
mock_mistral,
|
||||
[
|
||||
_connector(connector_id="c-1", name="mcp", tools=[_tool("a")]),
|
||||
_connector(connector_id="c-2", name="mcp", tools=[_tool("b")]),
|
||||
],
|
||||
)
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "mcp", "mcp_2")
|
||||
|
||||
assert "connector_mcp_a" in names
|
||||
assert "connector_mcp_2_b" in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_ready_connector_offers_no_tools(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(
|
||||
mock_mistral,
|
||||
[
|
||||
_connector(
|
||||
name="linear",
|
||||
is_ready=False,
|
||||
tools=[_tool("search")],
|
||||
auth_action={"type": "oauth"},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "linear")
|
||||
|
||||
assert not any(name.startswith("connector_linear") for name in names)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_connector_tools_are_withheld(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
_set_bootstrap(mock_mistral, [_connector(name="wiki", tools=[_tool("search")])])
|
||||
|
||||
# No ConnectorConfig entry → the connector stays disabled by default.
|
||||
names = await _offered_tool_names(mistral_api)
|
||||
|
||||
assert "connector_wiki_search" not in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_failure_still_lets_the_agent_run(
|
||||
mistral_api: MistralAPI, mock_mistral: respx.MockRouter
|
||||
) -> None:
|
||||
mock_mistral.get(CONNECTORS_BOOTSTRAP_PATH).mock(return_value=httpx.Response(500))
|
||||
|
||||
names = await _offered_tool_names(mistral_api, "wiki")
|
||||
|
||||
assert not any(name.startswith("connector_") for name in names)
|
||||
144
tests/agent_loop/e2e/test_e2e_fork_and_plan.py
Normal file
144
tests/agent_loop/e2e/test_e2e_fork_and_plan.py
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from vibe.core.agents.models import BuiltinAgentName
|
||||
from vibe.core.tools.builtins.ask_user_question import (
|
||||
AskUserQuestionArgs,
|
||||
AskUserQuestionResult,
|
||||
)
|
||||
from vibe.core.types import PlanReviewEndedEvent, PlanReviewRequestedEvent, Role
|
||||
from vibe.core.utils.tags import VIBE_WARNING_TAG
|
||||
|
||||
EXIT_PLAN_TOOL_CALL = [
|
||||
{
|
||||
"id": "call_exit_plan_1",
|
||||
"function": {"name": "exit_plan_mode", "arguments": "{}"},
|
||||
"index": 0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _user_message_ids(agent: Any) -> list[str]:
|
||||
return [m.message_id for m in agent.messages if m.role == Role.user]
|
||||
|
||||
|
||||
def _assistant_message_id(agent: Any) -> str:
|
||||
return next(m.message_id for m in agent.messages if m.role == Role.assistant)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_copies_all_non_system_messages(mistral_api: MistralAPI) -> None:
|
||||
# fork() with no anchor clones every non-system message into the child loop.
|
||||
mistral_api.reply(mistral_completion("Hi there!"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
forked = await agent.fork()
|
||||
|
||||
non_system = [m for m in forked.messages if m.role != Role.system]
|
||||
assert [m.role for m in non_system] == [Role.user, Role.assistant]
|
||||
assert forked.parent_session_id == agent.session_id
|
||||
assert forked.session_id != agent.session_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_from_message_id_truncates_at_next_user_turn(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# Forking from a user message keeps that turn but drops everything from the next one.
|
||||
mistral_api.reply(mistral_completion("First"), mistral_completion("Second"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Turn one")]
|
||||
_ = [event async for event in agent.act("Turn two")]
|
||||
|
||||
first_user_id = _user_message_ids(agent)[0]
|
||||
forked = await agent.fork(first_user_id)
|
||||
|
||||
contents = [m.content for m in forked.messages if m.role != Role.system]
|
||||
assert contents == ["Turn one", "First"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_from_unknown_message_id_raises(mistral_api: MistralAPI) -> None:
|
||||
# An unknown anchor id is rejected rather than silently forking everything.
|
||||
mistral_api.reply(mistral_completion("Hi"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
with pytest.raises(ValueError, match="unknown message_id"):
|
||||
await agent.fork("does-not-exist")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fork_from_assistant_message_id_raises(mistral_api: MistralAPI) -> None:
|
||||
# Forking is only allowed from user turns; an assistant anchor is rejected.
|
||||
mistral_api.reply(mistral_completion("Hi"))
|
||||
agent = build_e2e_agent_loop()
|
||||
_ = [event async for event in agent.act("Hello")]
|
||||
|
||||
assistant_id = _assistant_message_id(agent)
|
||||
with pytest.raises(ValueError, match="only supported for user messages"):
|
||||
await agent.fork(assistant_id)
|
||||
|
||||
|
||||
def _plan_agent(mistral_api: MistralAPI) -> Any:
|
||||
mistral_api.reply(
|
||||
mistral_completion("", tool_calls=EXIT_PLAN_TOOL_CALL),
|
||||
mistral_completion("Staying in plan mode."),
|
||||
)
|
||||
return build_e2e_agent_loop(
|
||||
config=e2e_config(enabled_tools=["exit_plan_mode"]),
|
||||
agent_name=BuiltinAgentName.PLAN,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plan_mode_emits_review_requested_and_ended_events(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# An exit_plan_mode round trip brackets the tool with review-requested/ended events.
|
||||
agent = _plan_agent(mistral_api)
|
||||
|
||||
async def stay_in_plan(_: AskUserQuestionArgs) -> AskUserQuestionResult:
|
||||
return AskUserQuestionResult(cancelled=True, answers=[])
|
||||
|
||||
agent.set_user_input_callback(stay_in_plan)
|
||||
|
||||
events = [event async for event in agent.act("Make a plan")]
|
||||
|
||||
assert any(isinstance(e, PlanReviewRequestedEvent) for e in events)
|
||||
assert any(isinstance(e, PlanReviewEndedEvent) for e in events)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plan_mode_injects_updated_plan_when_file_changed(
|
||||
mistral_api: MistralAPI,
|
||||
) -> None:
|
||||
# If the plan file changes during review, its new content is injected back as context.
|
||||
agent = _plan_agent(mistral_api)
|
||||
|
||||
plan_path: Path | None = None
|
||||
|
||||
async def edit_plan_then_decline(_: AskUserQuestionArgs) -> AskUserQuestionResult:
|
||||
assert plan_path is not None
|
||||
plan_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
plan_path.write_text("# Updated plan\nStep 1")
|
||||
return AskUserQuestionResult(cancelled=True, answers=[])
|
||||
|
||||
agent.set_user_input_callback(edit_plan_then_decline)
|
||||
|
||||
async for event in agent.act("Make a plan"):
|
||||
if isinstance(event, PlanReviewRequestedEvent):
|
||||
plan_path = event.file_path
|
||||
|
||||
injected = [m for m in agent.messages if getattr(m, "injected", False)]
|
||||
assert any(
|
||||
m.content and VIBE_WARNING_TAG in m.content and "# Updated plan" in m.content
|
||||
for m in injected
|
||||
)
|
||||
178
tests/agent_loop/e2e/test_e2e_provider_apis.py
Normal file
178
tests/agent_loop/e2e/test_e2e_provider_apis.py
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import (
|
||||
ProviderAPI,
|
||||
anthropic_e2e_config,
|
||||
assistant_text,
|
||||
build_e2e_agent_loop,
|
||||
openai_responses_e2e_config,
|
||||
)
|
||||
from tests.backend.data.anthropic import (
|
||||
anthropic_message,
|
||||
anthropic_reasoning_tool_use_stream,
|
||||
anthropic_request_content_blocks,
|
||||
anthropic_text_stream,
|
||||
anthropic_tool_use,
|
||||
)
|
||||
from tests.backend.data.openai_responses import (
|
||||
openai_function_call_item,
|
||||
openai_message_item,
|
||||
openai_reasoning_tool_call_stream,
|
||||
openai_response,
|
||||
openai_text_stream,
|
||||
)
|
||||
from vibe.core.types import ToolResultEvent
|
||||
|
||||
|
||||
class TestAnthropic:
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_answers(self, anthropic_api: ProviderAPI) -> None:
|
||||
# A plain prompt is answered through the Anthropic messages wire.
|
||||
anthropic_api.reply(anthropic_message("pong"))
|
||||
agent = build_e2e_agent_loop(config=anthropic_e2e_config())
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
assert agent.stats.context_tokens == 15
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams(self, anthropic_api: ProviderAPI) -> None:
|
||||
# Anthropic SSE deltas are reassembled into the final assistant content.
|
||||
anthropic_api.reply_stream(anthropic_text_stream("pong"))
|
||||
agent = build_e2e_agent_loop(
|
||||
config=anthropic_e2e_config(), enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_executes_tool_call(self, anthropic_api: ProviderAPI) -> None:
|
||||
# A tool_use turn runs the tool, then Anthropic returns the final answer.
|
||||
anthropic_api.reply(
|
||||
anthropic_tool_use("todo", {"action": "read"}),
|
||||
anthropic_message("Your list is empty."),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=anthropic_e2e_config(enabled_tools=["todo"])
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams_reasoning_and_tool_call(
|
||||
self, anthropic_api: ProviderAPI
|
||||
) -> None:
|
||||
# A streamed thinking + tool_use turn runs the tool, then replays that
|
||||
# reasoning/tool history into the follow-up Anthropic request.
|
||||
anthropic_api.reply_streams(
|
||||
anthropic_reasoning_tool_use_stream("todo", '{"action": "read"}'),
|
||||
anthropic_text_stream("Your list is empty."),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=anthropic_e2e_config(enabled_tools=["todo"]), enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
blocks = anthropic_request_content_blocks(anthropic_api.request_json)
|
||||
thinking = [b["thinking"] for b in blocks if b.get("type") == "thinking"]
|
||||
tool_uses = [b for b in blocks if b.get("type") == "tool_use"]
|
||||
|
||||
assert any("thinking..." in text for text in thinking)
|
||||
assert tool_uses
|
||||
|
||||
|
||||
class TestOpenAIResponses:
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_answers(self, openai_responses_api: ProviderAPI) -> None:
|
||||
# A plain prompt is answered through the OpenAI Responses wire.
|
||||
openai_responses_api.reply(openai_response([openai_message_item("pong")]))
|
||||
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
assert agent.stats.context_tokens == 12
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams(self, openai_responses_api: ProviderAPI) -> None:
|
||||
# OpenAI Responses SSE deltas are reassembled into the final content.
|
||||
openai_responses_api.reply_stream(openai_text_stream("pong"))
|
||||
agent = build_e2e_agent_loop(
|
||||
config=openai_responses_e2e_config(), enable_streaming=True
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_captures_reasoning(
|
||||
self, openai_responses_api: ProviderAPI
|
||||
) -> None:
|
||||
# Commentary phase output is captured as reasoning on the assistant message.
|
||||
openai_responses_api.reply(
|
||||
openai_response(
|
||||
[
|
||||
openai_message_item("Let me think.", phase="commentary"),
|
||||
openai_message_item("pong", phase="final_answer"),
|
||||
],
|
||||
output_tokens=5,
|
||||
)
|
||||
)
|
||||
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
|
||||
|
||||
events = [event async for event in agent.act("Reply with exactly: pong")]
|
||||
|
||||
assert assistant_text(events) == "pong"
|
||||
assert any(
|
||||
m.reasoning_content and "Let me think." in m.reasoning_content
|
||||
for m in agent.messages
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_executes_tool_call(
|
||||
self, openai_responses_api: ProviderAPI
|
||||
) -> None:
|
||||
# A function_call item runs the tool, then OpenAI returns the final answer.
|
||||
openai_responses_api.reply(
|
||||
openai_response([openai_function_call_item("todo", '{"action": "read"}')]),
|
||||
openai_response([openai_message_item("Your list is empty.")]),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=openai_responses_e2e_config(enabled_tools=["todo"])
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_streams_reasoning_and_tool_call(
|
||||
self, openai_responses_api: ProviderAPI
|
||||
) -> None:
|
||||
# A streamed commentary + function_call turn runs the tool, then OpenAI
|
||||
# streams the final answer on the follow-up request.
|
||||
openai_responses_api.reply_streams(
|
||||
openai_reasoning_tool_call_stream("todo", '{"action": "read"}'),
|
||||
openai_text_stream("Your list is empty."),
|
||||
)
|
||||
agent = build_e2e_agent_loop(
|
||||
config=openai_responses_e2e_config(enabled_tools=["todo"]),
|
||||
enable_streaming=True,
|
||||
)
|
||||
|
||||
events = [event async for event in agent.act("What's on my todo list?")]
|
||||
|
||||
assert any(isinstance(e, ToolResultEvent) for e in events)
|
||||
assert "Your list is empty." in assistant_text(events)
|
||||
224
tests/agent_loop/e2e/test_e2e_teleport.py
Normal file
224
tests/agent_loop/e2e/test_e2e_teleport.py
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from git import Repo
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from tests.agent_loop.e2e.conftest import build_e2e_agent_loop
|
||||
from tests.constants import (
|
||||
CONNECTORS_BOOTSTRAP_PATH,
|
||||
MISTRAL_BASE_URL,
|
||||
TELEPORT_COMPLETE_URL,
|
||||
TELEPORT_SESSIONS_PATH,
|
||||
)
|
||||
from vibe.core.agent_loop import AgentLoop, TeleportError
|
||||
from vibe.core.teleport.types import (
|
||||
TeleportCheckingGitEvent,
|
||||
TeleportCompleteEvent,
|
||||
TeleportPushingEvent,
|
||||
TeleportPushRequiredEvent,
|
||||
TeleportPushResponseEvent,
|
||||
TeleportStartingWorkflowEvent,
|
||||
)
|
||||
|
||||
# vibe_code_sessions_base_url is an exclude=True field, so it is dropped when the
|
||||
# agent manager re-derives config via model_dump(); the default url is unavoidable.
|
||||
SESSIONS_BASE_URL = "https://chat.mistral.ai"
|
||||
SESSIONS_URL = f"{SESSIONS_BASE_URL}{TELEPORT_SESSIONS_PATH}"
|
||||
|
||||
|
||||
def _sessions_ok() -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"sessionId": "controller-session-id",
|
||||
"webSessionId": "web-session-id",
|
||||
"projectId": "project-id",
|
||||
"status": "running",
|
||||
"url": TELEPORT_COMPLETE_URL,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _commit(repo: Repo, message: str) -> str:
|
||||
(Path(repo.working_dir) / "file.txt").write_text(f"{message}\n")
|
||||
repo.index.add(["file.txt"])
|
||||
repo.index.commit(message)
|
||||
return repo.head.commit.hexsha
|
||||
|
||||
|
||||
def _init_repo(workdir: Path) -> Repo:
|
||||
# origin is a local bare repo so fetch/push stay offline and instant; a
|
||||
# separate github-url remote satisfies Teleport's GitHub-only detection.
|
||||
bare = Repo.init(workdir.with_name(f"{workdir.name}_origin.git"), bare=True)
|
||||
repo = Repo.init(workdir, initial_branch="work")
|
||||
repo.config_writer().set_value("user", "name", "Tester").release()
|
||||
repo.config_writer().set_value("user", "email", "t@example.com").release()
|
||||
repo.create_remote("origin", str(bare.git_dir))
|
||||
repo.create_remote("hub", "https://github.com/owner/repo.git")
|
||||
return repo
|
||||
|
||||
|
||||
def _repo_with_pushed_branch(workdir: Path) -> Repo:
|
||||
repo = _init_repo(workdir)
|
||||
_commit(repo, "initial")
|
||||
repo.git.push("origin", "work")
|
||||
return repo
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_sessions() -> Iterator[respx.MockRouter]:
|
||||
# One router stubs both hosts: the connector bootstrap (api.mistral.ai, hit on
|
||||
# agent init) and the teleport sessions endpoint (chat.mistral.ai).
|
||||
with respx.mock(assert_all_called=False) as router:
|
||||
router.get(f"{MISTRAL_BASE_URL}{CONNECTORS_BOOTSTRAP_PATH}").mock(
|
||||
return_value=httpx.Response(200, json={"connectors": []})
|
||||
)
|
||||
router.post(SESSIONS_URL).mock(return_value=_sessions_ok())
|
||||
yield router
|
||||
|
||||
|
||||
async def _drain(
|
||||
agent: AgentLoop, prompt: str | None, *, approve: bool | None = None
|
||||
) -> list[object]:
|
||||
gen = agent.teleport_to_vibe_code(prompt)
|
||||
events: list[object] = []
|
||||
response: TeleportPushResponseEvent | None = None
|
||||
while True:
|
||||
try:
|
||||
event = await gen.asend(response)
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
events.append(event)
|
||||
response = None
|
||||
if isinstance(event, TeleportPushRequiredEvent) and approve is not None:
|
||||
response = TeleportPushResponseEvent(approved=approve)
|
||||
return events
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_completes_when_branch_already_pushed(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
_repo_with_pushed_branch(tmp_working_directory)
|
||||
|
||||
events = await _drain(build_e2e_agent_loop(), "do the thing")
|
||||
|
||||
assert [type(e) for e in events] == [
|
||||
TeleportCheckingGitEvent,
|
||||
TeleportStartingWorkflowEvent,
|
||||
TeleportCompleteEvent,
|
||||
]
|
||||
assert isinstance(events[-1], TeleportCompleteEvent)
|
||||
assert events[-1].url == TELEPORT_COMPLETE_URL
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_sends_repo_metadata_and_diff(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
commit = repo.head.commit.hexsha
|
||||
(tmp_working_directory / "file.txt").write_text("uncommitted change\n")
|
||||
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
|
||||
payload = mock_sessions.calls.last.request.read().decode()
|
||||
assert "https://github.com/owner/repo.git" in payload
|
||||
assert "work" in payload
|
||||
assert commit in payload
|
||||
assert "zstd" in payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_pushes_then_completes_when_approved(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
head = _commit(repo, "second")
|
||||
|
||||
events = await _drain(build_e2e_agent_loop(), "ship it", approve=True)
|
||||
|
||||
assert [type(e) for e in events] == [
|
||||
TeleportCheckingGitEvent,
|
||||
TeleportPushRequiredEvent,
|
||||
TeleportPushingEvent,
|
||||
TeleportStartingWorkflowEvent,
|
||||
TeleportCompleteEvent,
|
||||
]
|
||||
assert repo.remote("origin").refs["work"].commit.hexsha == head
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_aborts_when_push_declined(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
_commit(repo, "second")
|
||||
|
||||
with pytest.raises(TeleportError, match="not pushed"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it", approve=False)
|
||||
|
||||
assert not mock_sessions.post(SESSIONS_URL).called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_fails_when_push_fails(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
_commit(repo, "second")
|
||||
repo.git.remote("set-url", "--push", "origin", "/nonexistent/repo.git")
|
||||
|
||||
with pytest.raises(TeleportError, match="Failed to push"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it", approve=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_requires_a_branch(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _repo_with_pushed_branch(tmp_working_directory)
|
||||
repo.git.checkout(repo.head.commit.hexsha) # detach HEAD
|
||||
|
||||
with pytest.raises(TeleportError, match="checked-out branch"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_rejects_empty_prompt(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = _init_repo(tmp_working_directory)
|
||||
_commit(repo, "initial")
|
||||
|
||||
with pytest.raises(TeleportError, match="non-empty prompt"):
|
||||
await _drain(build_e2e_agent_loop(), None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_surfaces_http_error(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
_repo_with_pushed_branch(tmp_working_directory)
|
||||
mock_sessions.post(SESSIONS_URL).mock(return_value=httpx.Response(500, text="boom"))
|
||||
|
||||
with pytest.raises(TeleportError, match="start failed"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_teleport_unsupported_without_github_remote(
|
||||
tmp_working_directory: Path, mock_sessions: respx.MockRouter
|
||||
) -> None:
|
||||
repo = Repo.init(tmp_working_directory, initial_branch="work")
|
||||
repo.config_writer().set_value("user", "name", "Tester").release()
|
||||
repo.config_writer().set_value("user", "email", "t@example.com").release()
|
||||
_commit(repo, "initial")
|
||||
|
||||
with pytest.raises(TeleportError, match="GitHub"):
|
||||
await _drain(build_e2e_agent_loop(), "ship it")
|
||||
93
tests/agent_loop/e2e/test_e2e_tools.py
Normal file
93
tests/agent_loop/e2e/test_e2e_tools.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
|
||||
from tests.backend.data.mistral import mistral_completion
|
||||
from vibe.core.tools.builtins.grep import GrepResult
|
||||
from vibe.core.tools.builtins.read import ReadResult
|
||||
from vibe.core.tools.builtins.todo import TodoResult
|
||||
from vibe.core.types import BaseEvent, ToolResultEvent
|
||||
|
||||
|
||||
def _tool_call(name: str, arguments: dict[str, Any]) -> dict[str, Any]:
|
||||
return mistral_completion(
|
||||
"",
|
||||
tool_calls=[
|
||||
{
|
||||
"id": f"call_{name}",
|
||||
"function": {"name": name, "arguments": json.dumps(arguments)},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
async def _run_tool(
|
||||
mistral_api: MistralAPI, name: str, arguments: dict[str, Any]
|
||||
) -> ToolResultEvent:
|
||||
# Drive one tool call against the real cwd, then a final reply, and return
|
||||
# the single tool result for the test to assert on.
|
||||
mistral_api.reply(_tool_call(name, arguments), mistral_completion("done"))
|
||||
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=[name]))
|
||||
events: list[BaseEvent] = [event async for event in agent.act("go")]
|
||||
result = next(e for e in events if isinstance(e, ToolResultEvent))
|
||||
assert result.error is None
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_file_tool_creates_file(mistral_api: MistralAPI) -> None:
|
||||
target = Path.cwd() / "note.txt"
|
||||
|
||||
await _run_tool(mistral_api, "write_file", {"path": str(target), "content": "hi\n"})
|
||||
|
||||
assert target.read_text() == "hi\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_tool_returns_file_content(mistral_api: MistralAPI) -> None:
|
||||
target = Path.cwd() / "note.txt"
|
||||
target.write_text("hello world\n")
|
||||
|
||||
result = await _run_tool(mistral_api, "read", {"file_path": str(target)})
|
||||
|
||||
assert "hello world" in cast(ReadResult, result.result).content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_edit_tool_replaces_text(mistral_api: MistralAPI) -> None:
|
||||
target = Path.cwd() / "note.txt"
|
||||
target.write_text("hello world\n")
|
||||
|
||||
await _run_tool(
|
||||
mistral_api,
|
||||
"edit",
|
||||
{"file_path": str(target), "old_string": "hello", "new_string": "goodbye"},
|
||||
)
|
||||
|
||||
assert target.read_text() == "goodbye world\n"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_grep_tool_finds_matches(mistral_api: MistralAPI) -> None:
|
||||
(Path.cwd() / "note.txt").write_text("needle here\n")
|
||||
|
||||
result = await _run_tool(mistral_api, "grep", {"pattern": "needle", "path": "."})
|
||||
|
||||
assert cast(GrepResult, result.result).match_count >= 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_todo_tool_writes_items(mistral_api: MistralAPI) -> None:
|
||||
result = await _run_tool(
|
||||
mistral_api,
|
||||
"todo",
|
||||
{"action": "write", "todos": [{"id": "1", "content": "ship it"}]},
|
||||
)
|
||||
|
||||
assert cast(TodoResult, result.result).total_count == 1
|
||||
Loading…
Add table
Add a link
Reference in a new issue