uvos/vibe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2.18.0 (#843)

Browse source 
Co-authored-by: Clément Drouin <clement.drouin@mistral.ai>
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Cyprien <courtot.c@gmail.com>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com>
Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Nelson PROIA <144663685+Nelson-PROIA@users.noreply.github.com>
Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Quentin <quentin.torroba@mistral.ai>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: josephine-delas <57808586+josephine-delas@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Laure Hugo 2026-06-25 16:08:45 +02:00 • committed by GitHub
parent 725d3a56ce
commit e607ccbb00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
242 changed files with 7372 additions and 1974 deletions
Download patch file Download diff file Expand all files Collapse all files

153
tests/agent_loop/e2e/conftest.py
View file

@ -1,34 +1,17 @@
from __future__ import annotations
from collections.abc import Iterator
import json
from typing import Any
import httpx
import pytest
import respx
from tests.agent_loop.e2e.providers import MistralAPI
from tests.conftest import build_test_agent_loop, build_test_vibe_config
from tests.constants import (
ANTHROPIC_BASE_URL,
ANTHROPIC_MESSAGES_PATH,
CHAT_COMPLETIONS_PATH,
CONNECTORS_BOOTSTRAP_PATH,
MISTRAL_BASE_URL,
OPENAI_BASE_URL,
OPENAI_RESPONSES_PATH,
)
from vibe.core.agent_loop import AgentLoop
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.config import VibeConfig
from vibe.core.llm.backend.factory import BACKEND_FACTORY
from vibe.core.types import AssistantEvent, Backend, BaseEvent
def assistant_text(events: list[BaseEvent]) -> str:
return "".join(
e.content for e in events if isinstance(e, AssistantEvent) and e.content
).strip()
@pytest.fixture(autouse=True)
@ -38,71 +21,23 @@ def _git_offline(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("GIT_ALLOW_PROTOCOL", "file")
def e2e_config(**overrides: Any) -> VibeConfig:
# Defaults give the mistral provider/model (api.mistral.ai, the respx-mocked
# base). The default model reasons, but that only adds reasoning_effort to the
# request, which the mocked responses ignore.
# The e2e harness mocks the connector bootstrap endpoint via respx, so
# connector discovery is enabled (and fast) here unlike the unit-test default.
return build_test_vibe_config(
enabled_tools=overrides.pop("enabled_tools", []),
enable_connectors=overrides.pop("enable_connectors", True),
system_prompt_id="tests",
include_project_context=False,
include_prompt_detail=False,
**overrides,
)
def _generic_e2e_config(
*, name: str, api_base: str, api_style: str, model: str, **overrides: Any
) -> VibeConfig:
provider = ProviderConfig(
name=name,
api_base=api_base,
api_key_env_var=f"{name.upper()}_API_KEY",
api_style=api_style,
backend=Backend.GENERIC,
)
models = [ModelConfig(name=model, provider=name, alias=name)]
return e2e_config(
active_model=name, models=models, providers=[provider], **overrides
)
def anthropic_e2e_config(**overrides: Any) -> VibeConfig:
return _generic_e2e_config(
name="anthropic",
api_base=ANTHROPIC_BASE_URL,
api_style="anthropic",
model="claude-test",
**overrides,
)
def openai_responses_e2e_config(**overrides: Any) -> VibeConfig:
return _generic_e2e_config(
name="openai",
api_base=f"{OPENAI_BASE_URL}/v1",
api_style="openai-responses",
model="gpt-test",
**overrides,
)
@pytest.fixture
def mock_mistral() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=MistralAPI.base_url, assert_all_called=False) as router:
yield router
@pytest.fixture
def mock_mistral() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=MISTRAL_BASE_URL, assert_all_called=False) as router:
router.get(CONNECTORS_BOOTSTRAP_PATH).mock(
return_value=httpx.Response(200, json={"connectors": []})
)
yield router
def mistral_api(mock_mistral: respx.MockRouter) -> MistralAPI:
api = MistralAPI()
api.setup_router(mock_mistral)
return api
def build_e2e_agent_loop(
*, config: VibeConfig | None = None, enable_streaming: bool = False, **kwargs: Any
) -> AgentLoop:
resolved_config = config or e2e_config()
resolved_config = config or build_test_vibe_config()
provider = resolved_config.providers[0]
# todo now we use build_test_agent_loop for the fake mcp registry.
# Maybe mock http tool calls and instantiate a real registry later for more coverage
@ -113,69 +48,3 @@ def build_e2e_agent_loop(
enable_streaming=enable_streaming,
**kwargs,
)
class ProviderAPI:
"""Stubs a provider's completion wire; the AgentLoop stays the subject."""
def __init__(self, router: respx.MockRouter, path: str) -> None:
self.route = router.post(path)
def reply(self, *completions: dict[str, Any]) -> None:
responses = [httpx.Response(200, json=c) for c in completions]
if len(responses) == 1:
self.route.mock(return_value=responses[0])
else:
self.route.mock(side_effect=responses)
def reply_stream(self, chunks: list[bytes]) -> None:
self.route.mock(return_value=self._stream_response(chunks))
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
self.route.mock(
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
)
@staticmethod
def _stream_response(chunks: list[bytes]) -> httpx.Response:
return httpx.Response(
200,
stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
headers={"Content-Type": "text/event-stream"},
)
@property
def request_json(self) -> dict[str, Any]:
return json.loads(self.route.calls.last.request.content)
class MistralAPI(ProviderAPI):
def __init__(self, router: respx.MockRouter) -> None:
super().__init__(router, CHAT_COMPLETIONS_PATH)
@pytest.fixture
def mistral_api(mock_mistral: respx.MockRouter) -> MistralAPI:
return MistralAPI(mock_mistral)
@pytest.fixture
def mock_anthropic() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=ANTHROPIC_BASE_URL, assert_all_called=False) as router:
yield router
@pytest.fixture
def anthropic_api(mock_anthropic: respx.MockRouter) -> ProviderAPI:
return ProviderAPI(mock_anthropic, ANTHROPIC_MESSAGES_PATH)
@pytest.fixture
def mock_openai() -> Iterator[respx.MockRouter]:
with respx.mock(base_url=OPENAI_BASE_URL, assert_all_called=False) as router:
yield router
@pytest.fixture
def openai_responses_api(mock_openai: respx.MockRouter) -> ProviderAPI:
return ProviderAPI(mock_openai, OPENAI_RESPONSES_PATH)

10
tests/agent_loop/e2e/providers/__init__.py Normal file
View file

@ -0,0 +1,10 @@
from __future__ import annotations
from tests.agent_loop.e2e.providers.base import (
ProviderAPI,
ProviderMocks,
assistant_text,
)
from tests.agent_loop.e2e.providers.mistral import MistralAPI
__all__ = ["MistralAPI", "ProviderAPI", "ProviderMocks", "assistant_text"]

61
tests/agent_loop/e2e/providers/anthropic.py Normal file
View file

@ -0,0 +1,61 @@
from __future__ import annotations
import json
from typing import Any
from tests import constants as c
from tests.agent_loop.e2e.providers.base import ProviderAPI
from tests.backend.data import Chunk, JsonResponse
from tests.backend.data.anthropic import (
anthropic_message,
anthropic_reasoning_tool_use_stream,
anthropic_text_stream,
anthropic_tool_use,
)
from tests.conftest import build_test_vibe_config
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.types import Backend
def e2e_config(**overrides: Any) -> VibeConfig:
provider = ProviderConfig(
name="anthropic",
api_base=c.ANTHROPIC_BASE_URL,
api_key_env_var="ANTHROPIC_API_KEY",
api_style="anthropic",
backend=Backend.GENERIC,
)
models = [ModelConfig(name="claude-test", provider="anthropic", alias="anthropic")]
return build_test_vibe_config(
active_model="anthropic", models=models, providers=[provider], **overrides
)
class API(ProviderAPI):
base_url = c.ANTHROPIC_BASE_URL
post_path = c.ANTHROPIC_MESSAGES_PATH
class Mocks:
def answer(self, text: str) -> JsonResponse:
return anthropic_message(text)
def text_stream(self, text: str) -> list[Chunk]:
return anthropic_text_stream(text)
def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
return anthropic_tool_use(name, arguments)
def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
message = anthropic_message(text)
message["content"].insert(
0, {"type": "thinking", "thinking": reasoning, "signature": "sig"}
)
return message
def reasoning_tool_call_stream(
self, name: str, arguments: dict[str, Any], *, reasoning: str
) -> list[Chunk]:
return anthropic_reasoning_tool_use_stream(
name, json.dumps(arguments), reasoning=reasoning
)

98
tests/agent_loop/e2e/providers/base.py Normal file
View file

@ -0,0 +1,98 @@
from __future__ import annotations
import json
from typing import Any, ClassVar, Protocol
import httpx
import pytest
import respx
from tests.backend.data import Chunk, JsonResponse
from vibe.core.types import AssistantEvent, BaseEvent
def assistant_text(events: list[BaseEvent]) -> str:
return "".join(
e.content for e in events if isinstance(e, AssistantEvent) and e.content
).strip()
class ProviderMocks(Protocol):
"""Canonical wire payloads a provider must produce for the shared e2e suite."""
def answer(self, text: str) -> JsonResponse:
"""A non-streaming assistant turn that replies with `text`."""
...
def text_stream(self, text: str) -> list[Chunk]:
"""A streaming assistant turn whose deltas reassemble into `text`."""
...
def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
"""A non-streaming turn that calls tool `name` with `arguments`."""
...
def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
"""A non-streaming turn that thinks `reasoning`, then replies `text`."""
...
def reasoning_tool_call_stream(
self, name: str, arguments: dict[str, Any], *, reasoning: str
) -> list[Chunk]:
"""A streamed turn that thinks `reasoning`, then calls tool `name`."""
...
class ProviderAPI:
"""Stubs a provider's completion wire; the AgentLoop stays the subject.
Subclasses describe one provider via `base_url`/`post_path`; override
`setup_monkeypatch` for non-HTTP stubs (e.g. credentials) a provider needs.
"""
base_url: ClassVar[str]
post_path: ClassVar[str]
route: respx.Route
def setup_router(self, router: respx.MockRouter) -> None:
"""Bind the completion route; subclasses extend for extra wires/defaults."""
self.route = router.post(self.post_path)
@classmethod
def setup_monkeypatch(cls, monkeypatch: pytest.MonkeyPatch) -> None:
"""Apply non-HTTP stubs (e.g. credential lookups) before use."""
def reply(self, *completions: dict[str, Any]) -> None:
responses = [httpx.Response(200, json=completion) for completion in completions]
if len(responses) == 1:
self.route.mock(return_value=responses[0])
else:
self.route.mock(side_effect=responses)
def reply_stream(self, chunks: list[bytes]) -> None:
self.route.mock(return_value=self._stream_response(chunks))
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
self.route.mock(
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
)
@staticmethod
def _stream_response(chunks: list[bytes]) -> httpx.Response:
return httpx.Response(
200,
stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
headers={"Content-Type": "text/event-stream"},
)
@property
def request_json(self) -> dict[str, Any]:
return json.loads(self.route.calls.last.request.content)
def model_facing_text(self, index: int) -> str:
"""Return the text (raw prompt) that the model would have seen in the request
at the given call index.
"""
body = json.loads(self.route.calls[index].request.content)
return json.dumps([m for m in body["messages"] if m.get("role") != "system"])

18
tests/agent_loop/e2e/providers/mistral.py Normal file
View file

@ -0,0 +1,18 @@
from __future__ import annotations
import httpx
import respx
from tests import constants as c
from tests.agent_loop.e2e.providers.base import ProviderAPI
class MistralAPI(ProviderAPI):
base_url = c.MISTRAL_BASE_URL
post_path = c.CHAT_COMPLETIONS_PATH
def setup_router(self, router: respx.MockRouter) -> None:
super().setup_router(router)
router.get(c.CONNECTORS_BOOTSTRAP_PATH).mock(
return_value=httpx.Response(200, json={"connectors": []})
)

61
tests/agent_loop/e2e/providers/openai_responses.py Normal file
View file

@ -0,0 +1,61 @@
from __future__ import annotations
import json
from typing import Any
from tests import constants as c
from tests.agent_loop.e2e.providers.base import ProviderAPI
from tests.backend.data import Chunk, JsonResponse
from tests.backend.data.openai_responses import (
openai_function_call_item,
openai_message_item,
openai_reasoning_tool_call_stream,
openai_response,
openai_text_stream,
)
from tests.conftest import build_test_vibe_config
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.types import Backend
def e2e_config(**overrides: Any) -> VibeConfig:
provider = ProviderConfig(
name="openai",
api_base=f"{c.OPENAI_BASE_URL}/v1",
api_key_env_var="OPENAI_API_KEY",
api_style="openai-responses",
backend=Backend.GENERIC,
)
models = [ModelConfig(name="gpt-test", provider="openai", alias="openai")]
return build_test_vibe_config(
active_model="openai", models=models, providers=[provider], **overrides
)
class API(ProviderAPI):
base_url = c.OPENAI_BASE_URL
post_path = c.OPENAI_RESPONSES_PATH
class Mocks:
def answer(self, text: str) -> JsonResponse:
return openai_response([openai_message_item(text)])
def text_stream(self, text: str) -> list[Chunk]:
return openai_text_stream(text)
def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
return openai_response([openai_function_call_item(name, json.dumps(arguments))])
def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
return openai_response([
openai_message_item(reasoning, phase="commentary"),
openai_message_item(text, phase="final_answer"),
])
def reasoning_tool_call_stream(
self, name: str, arguments: dict[str, Any], *, reasoning: str
) -> list[Chunk]:
return openai_reasoning_tool_call_stream(
name, json.dumps(arguments), reasoning=reasoning
)

65
tests/agent_loop/e2e/providers/reasoning.py Normal file
View file

@ -0,0 +1,65 @@
from __future__ import annotations
import json
from typing import Any
from tests import constants as c
from tests.agent_loop.e2e.providers.base import ProviderAPI
from tests.backend.data import Chunk, JsonResponse
from tests.backend.data.reasoning import (
reasoning_message,
reasoning_text_stream,
reasoning_thinking_message,
reasoning_thinking_tool_use_stream,
reasoning_tool_use,
)
from tests.conftest import build_test_vibe_config
from vibe.core.config import ModelConfig, ProviderConfig, ThinkingLevel, VibeConfig
from vibe.core.types import Backend
def e2e_config(*, thinking: ThinkingLevel = "off", **overrides: Any) -> VibeConfig:
provider = ProviderConfig(
name="reasoning",
api_base=c.REASONING_BASE_URL,
api_key_env_var="REASONING_API_KEY",
api_style="reasoning",
backend=Backend.GENERIC,
)
models = [
ModelConfig(
name="reasoning-test",
provider="reasoning",
alias="reasoning",
thinking=thinking,
)
]
return build_test_vibe_config(
active_model="reasoning", models=models, providers=[provider], **overrides
)
class API(ProviderAPI):
base_url = c.REASONING_BASE_URL
post_path = c.REASONING_COMPLETIONS_PATH
class Mocks:
def answer(self, text: str) -> JsonResponse:
return reasoning_message(text)
def text_stream(self, text: str) -> list[Chunk]:
return reasoning_text_stream(text)
def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
return reasoning_tool_use(name, json.dumps(arguments))
def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
return reasoning_thinking_message(text, reasoning)
def reasoning_tool_call_stream(
self, name: str, arguments: dict[str, Any], *, reasoning: str
) -> list[Chunk]:
return reasoning_thinking_tool_use_stream(
name, json.dumps(arguments), reasoning=reasoning
)

69
tests/agent_loop/e2e/providers/vertex.py Normal file
View file

@ -0,0 +1,69 @@
from __future__ import annotations
import json
from typing import Any
from unittest.mock import MagicMock
import pytest
import respx
from tests import constants as c
from tests.agent_loop.e2e.providers.anthropic import Mocks
from tests.agent_loop.e2e.providers.base import ProviderAPI
from tests.conftest import build_test_vibe_config
from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
from vibe.core.types import Backend
__all__ = ["API", "Mocks", "e2e_config"]
def e2e_config(**overrides: Any) -> VibeConfig:
provider = ProviderConfig(
name="vertex",
api_base="",
api_key_env_var="VERTEX_API_KEY",
api_style="vertex-anthropic",
backend=Backend.GENERIC,
project_id=c.VERTEX_PROJECT_ID,
region=c.VERTEX_REGION,
)
models = [ModelConfig(name=c.VERTEX_MODEL, provider="vertex", alias="vertex")]
return build_test_vibe_config(
active_model="vertex", models=models, providers=[provider], **overrides
)
class API(ProviderAPI):
"""Vertex splits rawPredict and streamRawPredict across two wire paths."""
base_url = c.VERTEX_BASE_URL
post_path = c.VERTEX_RAW_PREDICT_PATH
def setup_router(self, router: respx.MockRouter) -> None:
super().setup_router(router)
self._stream_route = router.post(c.VERTEX_STREAM_PREDICT_PATH)
@classmethod
def setup_monkeypatch(cls, monkeypatch: pytest.MonkeyPatch) -> None:
# google.auth is the only external dependency Vertex pulls in; stub the
# credential lookup so the real token-refresh logic runs without network.
creds = MagicMock()
creds.valid = True
creds.token = "fake-vertex-token"
monkeypatch.setattr(
"vibe.core.llm.backend.vertex.google.auth.default",
lambda **_kwargs: (creds, c.VERTEX_PROJECT_ID),
)
def reply_stream(self, chunks: list[bytes]) -> None:
self._stream_route.mock(return_value=self._stream_response(chunks))
def reply_streams(self, *chunk_lists: list[bytes]) -> None:
self._stream_route.mock(
side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
)
@property
def request_json(self) -> dict[str, Any]:
route = self._stream_route if self._stream_route.calls else self.route
return json.loads(route.calls.last.request.content)

13
tests/agent_loop/e2e/test_e2e_agent_loop.py
View file

@ -4,16 +4,13 @@ from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import (
MistralAPI,
assistant_text,
build_e2e_agent_loop,
e2e_config,
)
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.agent_loop.e2e.providers import assistant_text
from tests.backend.data.mistral import (
STREAMED_SIMPLE_CONVERSATION_PARAMS,
mistral_completion,
)
from tests.conftest import build_test_vibe_config
from vibe.core.types import (
AssistantEvent,
ToolCallEvent,
@ -75,7 +72,7 @@ async def test_act_tool_call_round_trip(mistral_api: MistralAPI) -> None:
mistral_completion("", tool_calls=TODO_TOOL_CALL),
mistral_completion("All done"),
)
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
agent = build_e2e_agent_loop(config=build_test_vibe_config(enabled_tools=["todo"]))
events = [event async for event in agent.act("Show my todos")]
@ -89,7 +86,7 @@ async def test_act_tool_call_round_trip(mistral_api: MistralAPI) -> None:
async def test_act_serializes_tools_in_request_payload(mistral_api: MistralAPI) -> None:
# Enabled tools are serialized into the outgoing chat-completions request.
mistral_api.reply(mistral_completion("ok"))
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=["todo"]))
agent = build_e2e_agent_loop(config=build_test_vibe_config(enabled_tools=["todo"]))
_ = [event async for event in agent.act("Hello")]

5
tests/agent_loop/e2e/test_e2e_bash.py
View file

@ -7,8 +7,9 @@ from typing import Any, cast
from pydantic import BaseModel
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.backend.data.mistral import mistral_completion
from tests.conftest import build_test_vibe_config
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.tools.builtins.bash import BashResult
from vibe.core.tools.permissions import RequiredPermission
@ -41,7 +42,7 @@ async def _run_bash(
) -> ToolResultEvent:
mistral_api.reply(_bash_call(command, timeout), mistral_completion("done"))
agent = build_e2e_agent_loop(
config=e2e_config(enabled_tools=["bash"]), agent_name=agent_name
config=build_test_vibe_config(enabled_tools=["bash"]), agent_name=agent_name
)
if approval is not None:

82
tests/agent_loop/e2e/test_e2e_compaction.py Normal file
View file

@ -0,0 +1,82 @@
from __future__ import annotations
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.backend.data.mistral import mistral_completion
from tests.conftest import build_test_vibe_config, make_test_models
from vibe.core.types import CompactStartEvent
COMPACTION_MODELS = make_test_models(auto_compact_threshold=1)
@pytest.mark.asyncio
async def test_auto_compaction_preserves_user_message_and_embeds_summary(
mistral_api: MistralAPI,
) -> None:
mistral_api.reply(
# First: the compaction summary
mistral_completion("A summary of what happened so far"),
# Then: the final answer to user message
mistral_completion("final answer"),
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(models=COMPACTION_MODELS)
)
agent.stats.context_tokens = 5 # Trigger the auto-compaction immediately
events = [event async for event in agent.act("Investigate the bug")]
assert any(isinstance(e, CompactStartEvent) for e in events)
sent_after_compaction = mistral_api.model_facing_text(1)
assert "Investigate the bug" in sent_after_compaction
assert "A summary of what happened so far" in sent_after_compaction
@pytest.mark.asyncio
async def test_repeated_auto_compaction_preserves_earlier_user_messages(
mistral_api: MistralAPI,
) -> None:
mistral_api.reply(
mistral_completion("summary one"),
mistral_completion("reply one"),
mistral_completion("summary two"),
mistral_completion("reply two"),
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(models=COMPACTION_MODELS)
)
agent.stats.context_tokens = 5
[_ async for _ in agent.act("first ask")]
agent.stats.context_tokens = 5
[_ async for _ in agent.act("second ask")]
sent_after_second_compaction = mistral_api.model_facing_text(3)
assert "first ask" in sent_after_second_compaction
assert "second ask" in sent_after_second_compaction
@pytest.mark.asyncio
async def test_oversized_user_message_is_middle_truncated_in_compaction(
mistral_api: MistralAPI,
) -> None:
huge_message = "alpha " * 20_000
mistral_api.reply(
mistral_completion("summary intro"),
mistral_completion("reply intro"),
mistral_completion("summary huge"),
mistral_completion("reply huge"),
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(models=COMPACTION_MODELS)
)
agent.stats.context_tokens = 5
[_ async for _ in agent.act("intro")]
agent.stats.context_tokens = 5
[_ async for _ in agent.act(huge_message)]
sent_after_second_compaction = mistral_api.model_facing_text(3)
assert "[... truncated ...]" in sent_after_second_compaction
assert "intro" not in sent_after_second_compaction

8
tests/agent_loop/e2e/test_e2e_connectors.py
View file

@ -6,8 +6,9 @@ import httpx
import pytest
import respx
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.backend.data.mistral import mistral_completion
from tests.conftest import build_test_vibe_config
from tests.constants import CONNECTORS_BOOTSTRAP_PATH
from vibe.core.config import ConnectorConfig
@ -44,8 +45,9 @@ def _set_bootstrap(router: respx.MockRouter, connectors: list[dict[str, Any]]) -
def _connectors_config(*aliases: str) -> Any:
return e2e_config(
connectors=[ConnectorConfig(name=alias, disabled=False) for alias in aliases]
return build_test_vibe_config(
enable_connectors=True,
connectors=[ConnectorConfig(name=alias, disabled=False) for alias in aliases],
)

5
tests/agent_loop/e2e/test_e2e_fork_and_plan.py
View file

@ -5,8 +5,9 @@ from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.backend.data.mistral import mistral_completion
from tests.conftest import build_test_vibe_config
from vibe.core.agents.models import BuiltinAgentName
from vibe.core.tools.builtins.ask_user_question import (
AskUserQuestionArgs,
@ -93,7 +94,7 @@ def _plan_agent(mistral_api: MistralAPI) -> Any:
mistral_completion("Staying in plan mode."),
)
return build_e2e_agent_loop(
config=e2e_config(enabled_tools=["exit_plan_mode"]),
config=build_test_vibe_config(enabled_tools=["exit_plan_mode"]),
agent_name=BuiltinAgentName.PLAN,
)

164
tests/agent_loop/e2e/test_e2e_hooks.py Normal file
View file

@ -0,0 +1,164 @@
from __future__ import annotations
import json
import shlex
import sys
import textwrap
from typing import Any
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.backend.data.mistral import mistral_completion
from tests.conftest import build_test_vibe_config
from vibe.core.hooks.config import HookConfigResult
from vibe.core.hooks.models import HookConfig, HookType
from vibe.core.types import AssistantEvent, ToolResultEvent
TODO_TOOL_CALL: list[dict[str, Any]] = [
{
"id": "call_todo",
"function": {"name": "todo", "arguments": '{"action": "read"}'},
"index": 0,
}
]
def _emit_cmd(payload: dict[str, Any]) -> str:
# Payload is passed via stdin of the runner (stdout of parent process)
body = f"import sys; sys.stdout.write({json.dumps(payload)!r})"
return f"{sys.executable} -c {shlex.quote(body)}"
@pytest.mark.asyncio
async def test_before_tool_hook_denies_tool_and_skips_execution(
mistral_api: MistralAPI,
) -> None:
deny_hook = HookConfig(
name="deny",
type=HookType.BEFORE_TOOL,
command=_emit_cmd({"decision": "deny", "reason": "blocked by policy"}),
match="todo",
)
mistral_api.reply(
mistral_completion("", tool_calls=TODO_TOOL_CALL), mistral_completion("done")
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(enabled_tools=["todo"]),
hook_config_result=HookConfigResult(hooks=[deny_hook], issues=[]),
)
events = [event async for event in agent.act("Show my todos")]
tool_result = next(e for e in events if isinstance(e, ToolResultEvent))
assert tool_result.skipped is True
assert "blocked by policy" in (tool_result.skip_reason or "")
@pytest.mark.asyncio
async def test_before_tool_hook_rewrites_tool_input_seen_by_model(
mistral_api: MistralAPI,
) -> None:
rewritten = {
"action": "write",
"todos": [{"id": "1", "content": "rewritten by hook"}],
}
rewrite_hook = HookConfig(
name="rewrite",
type=HookType.BEFORE_TOOL,
command=_emit_cmd({"hook_specific_output": {"tool_input": rewritten}}),
match="todo",
)
mistral_api.reply(
mistral_completion("", tool_calls=TODO_TOOL_CALL), mistral_completion("done")
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(enabled_tools=["todo"]),
hook_config_result=HookConfigResult(hooks=[rewrite_hook], issues=[]),
)
[_ async for _ in agent.act("Update my todos")]
assert "rewritten by hook" in mistral_api.model_facing_text(1)
@pytest.mark.asyncio
async def test_before_tool_rewrite_failing_validation_is_denied(
mistral_api: MistralAPI,
) -> None:
bad_rewrite_hook = HookConfig(
name="bad-rewrite",
type=HookType.BEFORE_TOOL,
command=_emit_cmd({"hook_specific_output": {"tool_input": {"todos": "x"}}}),
match="todo",
)
mistral_api.reply(
mistral_completion("", tool_calls=TODO_TOOL_CALL), mistral_completion("done")
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(enabled_tools=["todo"]),
hook_config_result=HookConfigResult(hooks=[bad_rewrite_hook], issues=[]),
)
events = [event async for event in agent.act("Update my todos")]
tool_result = next(e for e in events if isinstance(e, ToolResultEvent))
assert tool_result.skipped is True
assert "failed validation" in (tool_result.skip_reason or "")
@pytest.mark.asyncio
async def test_after_tool_hook_replaces_tool_output_seen_by_model(
mistral_api: MistralAPI,
) -> None:
replace_output_hook = HookConfig(
name="replace",
type=HookType.AFTER_TOOL,
command=_emit_cmd({"decision": "deny", "reason": "REPLACED OUTPUT"}),
match="todo",
)
mistral_api.reply(
mistral_completion("", tool_calls=TODO_TOOL_CALL), mistral_completion("done")
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(enabled_tools=["todo"]),
hook_config_result=HookConfigResult(hooks=[replace_output_hook], issues=[]),
)
[_ async for _ in agent.act("Show my todos")]
assert "REPLACED OUTPUT" in mistral_api.model_facing_text(1)
@pytest.mark.asyncio
async def test_post_agent_turn_hook_injects_retry_user_message(
mistral_api: MistralAPI,
) -> None:
# A post-agent-turn hook that denies forever would loop, so this script
# denies only on its first run, using a sentinel file to stay silent after.
deny_decision = json.dumps({"decision": "deny", "reason": "please continue"})
deny_once = textwrap.dedent(f"""
import os, sys
sentinel = "__post_turn_sentinel__"
if not os.path.exists(sentinel):
open(sentinel, "w").close()
sys.stdout.write({deny_decision!r})
""")
retry_hook = HookConfig(
name="retry",
type=HookType.POST_AGENT_TURN,
command=f"{sys.executable} -c {shlex.quote(deny_once)}",
)
mistral_api.reply(
mistral_completion("first answer"), mistral_completion("second answer")
)
agent = build_e2e_agent_loop(
config=build_test_vibe_config(enabled_tools=["todo"]),
hook_config_result=HookConfigResult(hooks=[retry_hook], issues=[]),
)
events = [event async for event in agent.act("Hello")]
assert "please continue" in mistral_api.model_facing_text(1)
answers = [e.content for e in events if isinstance(e, AssistantEvent)]
assert answers == ["first answer", "second answer"]

271
tests/agent_loop/e2e/test_e2e_provider_apis.py
View file

@ -1,135 +1,146 @@
from __future__ import annotations
import pytest
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from tests.agent_loop.e2e.conftest import (
import pytest
import respx
from tests.agent_loop.e2e.conftest import build_e2e_agent_loop
from tests.agent_loop.e2e.providers import (
anthropic,
openai_responses,
reasoning,
vertex,
)
from tests.agent_loop.e2e.providers.base import (
ProviderAPI,
anthropic_e2e_config,
ProviderMocks,
assistant_text,
build_e2e_agent_loop,
openai_responses_e2e_config,
)
from tests.backend.data.anthropic import (
anthropic_message,
anthropic_reasoning_tool_use_stream,
anthropic_request_content_blocks,
anthropic_text_stream,
anthropic_tool_use,
)
from tests.backend.data.openai_responses import (
openai_function_call_item,
openai_message_item,
openai_reasoning_tool_call_stream,
openai_response,
openai_text_stream,
)
from tests.backend.data import ANSWER_CONTEXT_TOKENS
from tests.backend.data.anthropic import anthropic_message
from tests.backend.data.reasoning import reasoning_thinking_message
from vibe.core.config import VibeConfig
from vibe.core.types import ToolResultEvent
class TestAnthropic:
@dataclass(frozen=True)
class ProviderScenario:
"""Everything the shared e2e suite needs to exercise one provider.
Adding a provider is: implement `ProviderMocks`, then append one entry here.
"""
id: str
config: Callable[..., VibeConfig]
api_cls: type[ProviderAPI]
mocks: ProviderMocks
PROVIDER_SCENARIOS: list[ProviderScenario] = [
ProviderScenario(
id="anthropic",
config=anthropic.e2e_config,
api_cls=anthropic.API,
mocks=anthropic.Mocks(),
),
ProviderScenario(
id="openai-responses",
config=openai_responses.e2e_config,
api_cls=openai_responses.API,
mocks=openai_responses.Mocks(),
),
ProviderScenario(
id="reasoning",
config=reasoning.e2e_config,
api_cls=reasoning.API,
mocks=reasoning.Mocks(),
),
ProviderScenario(
id="vertex-anthropic",
config=vertex.e2e_config,
api_cls=vertex.API,
mocks=vertex.Mocks(),
),
]
@contextmanager
def open_provider_api(
api_cls: type[ProviderAPI], request: pytest.FixtureRequest
) -> Iterator[ProviderAPI]:
"""Instantiate a provider API and setup respx routers and monkeypatches"""
api_cls.setup_monkeypatch(request.getfixturevalue("monkeypatch"))
with respx.mock(base_url=api_cls.base_url, assert_all_called=False) as router:
api = api_cls()
api.setup_router(router)
yield api
@pytest.fixture(params=PROVIDER_SCENARIOS, ids=lambda s: s.id)
def scenario(request: pytest.FixtureRequest) -> ProviderScenario:
"""Parametrized fixture for each provider scenario in the shared e2e suite."""
return request.param
@pytest.fixture
def provider_api(
scenario: ProviderScenario, request: pytest.FixtureRequest
) -> Iterator[ProviderAPI]:
"""Returned setup provider API for the current scenario"""
with open_provider_api(scenario.api_cls, request) as api:
yield api
class TestProviderCommonBehaviors:
"""Answer / stream / tool-call behaviors every provider shares."""
@pytest.mark.asyncio
async def test_agent_answers(self, anthropic_api: ProviderAPI) -> None:
# A plain prompt is answered through the Anthropic messages wire.
anthropic_api.reply(anthropic_message("pong"))
agent = build_e2e_agent_loop(config=anthropic_e2e_config())
async def test_agent_answers(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
provider_api.reply(scenario.mocks.answer("pong"))
agent = build_e2e_agent_loop(config=scenario.config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 15
assert agent.stats.context_tokens == ANSWER_CONTEXT_TOKENS
@pytest.mark.asyncio
async def test_agent_streams(self, anthropic_api: ProviderAPI) -> None:
# Anthropic SSE deltas are reassembled into the final assistant content.
anthropic_api.reply_stream(anthropic_text_stream("pong"))
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(), enable_streaming=True
)
async def test_agent_streams(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
provider_api.reply_stream(scenario.mocks.text_stream("pong"))
agent = build_e2e_agent_loop(config=scenario.config(), enable_streaming=True)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_executes_tool_call(self, anthropic_api: ProviderAPI) -> None:
# A tool_use turn runs the tool, then Anthropic returns the final answer.
anthropic_api.reply(
anthropic_tool_use("todo", {"action": "read"}),
anthropic_message("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"])
async def test_agent_executes_tool_call(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
provider_api.reply(
scenario.mocks.tool_call("todo", {"action": "read"}),
scenario.mocks.answer("Your list is empty."),
)
agent = build_e2e_agent_loop(config=scenario.config(enabled_tools=["todo"]))
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, anthropic_api: ProviderAPI
) -> None:
# A streamed thinking + tool_use turn runs the tool, then replays that
# reasoning/tool history into the follow-up Anthropic request.
anthropic_api.reply_streams(
anthropic_reasoning_tool_use_stream("todo", '{"action": "read"}'),
anthropic_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"]), enable_streaming=True
)
events = [event async for event in agent.act("What's on my todo list?")]
assert "Your list is empty." in assistant_text(events)
blocks = anthropic_request_content_blocks(anthropic_api.request_json)
thinking = [b["thinking"] for b in blocks if b.get("type") == "thinking"]
tool_uses = [b for b in blocks if b.get("type") == "tool_use"]
assert any("thinking..." in text for text in thinking)
assert tool_uses
class TestOpenAIResponses:
@pytest.mark.asyncio
async def test_agent_answers(self, openai_responses_api: ProviderAPI) -> None:
# A plain prompt is answered through the OpenAI Responses wire.
openai_responses_api.reply(openai_response([openai_message_item("pong")]))
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 12
@pytest.mark.asyncio
async def test_agent_streams(self, openai_responses_api: ProviderAPI) -> None:
# OpenAI Responses SSE deltas are reassembled into the final content.
openai_responses_api.reply_stream(openai_text_stream("pong"))
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(), enable_streaming=True
)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_captures_reasoning(
self, openai_responses_api: ProviderAPI
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
# Commentary phase output is captured as reasoning on the assistant message.
openai_responses_api.reply(
openai_response(
[
openai_message_item("Let me think.", phase="commentary"),
openai_message_item("pong", phase="final_answer"),
],
output_tokens=5,
)
)
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
provider_api.reply(scenario.mocks.reasoning_answer("pong", "Let me think."))
agent = build_e2e_agent_loop(config=scenario.config())
events = [event async for event in agent.act("Reply with exactly: pong")]
@ -140,39 +151,55 @@ class TestOpenAIResponses:
)
@pytest.mark.asyncio
async def test_agent_executes_tool_call(
self, openai_responses_api: ProviderAPI
async def test_agent_streams_reasoning_then_runs_tool(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
# A function_call item runs the tool, then OpenAI returns the final answer.
openai_responses_api.reply(
openai_response([openai_function_call_item("todo", '{"action": "read"}')]),
openai_response([openai_message_item("Your list is empty.")]),
provider_api.reply_streams(
scenario.mocks.reasoning_tool_call_stream(
"todo", {"action": "read"}, reasoning="thinking..."
),
scenario.mocks.text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"])
config=scenario.config(enabled_tools=["todo"]), enable_streaming=True
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
assert any(
m.reasoning_content and "thinking..." in m.reasoning_content
for m in agent.messages
)
# The following tests are provider-specific and not shared across all providers, so they are not parametrized by scenario.
class TestReasoning:
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, openai_responses_api: ProviderAPI
async def test_forwards_thinking_level_as_reasoning_effort(
self, request: pytest.FixtureRequest
) -> None:
# A streamed commentary + function_call turn runs the tool, then OpenAI
# streams the final answer on the follow-up request.
openai_responses_api.reply_streams(
openai_reasoning_tool_call_stream("todo", '{"action": "read"}'),
openai_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"]),
enable_streaming=True,
)
# The model's thinking level maps to reasoning_effort on the wire.
with open_provider_api(reasoning.API, request) as api:
api.reply(reasoning_thinking_message("pong", "Let me think."))
agent = build_e2e_agent_loop(config=reasoning.e2e_config(thinking="medium"))
events = [event async for event in agent.act("What's on my todo list?")]
async for _ in agent.act("Reply with exactly: pong"):
pass
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
assert api.request_json["reasoning_effort"] == "medium"
class TestVertexAnthropic:
@pytest.mark.asyncio
async def test_uses_vertex_wire(self, request: pytest.FixtureRequest) -> None:
# The request goes out on the Vertex rawPredict wire format.
with open_provider_api(vertex.API, request) as api:
api.reply(anthropic_message("pong"))
agent = build_e2e_agent_loop(config=vertex.e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert api.request_json["anthropic_version"] == "vertex-2023-10-16"

5
tests/agent_loop/e2e/test_e2e_tools.py
View file

@ -6,8 +6,9 @@ from typing import Any, cast
import pytest
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop, e2e_config
from tests.agent_loop.e2e.conftest import MistralAPI, build_e2e_agent_loop
from tests.backend.data.mistral import mistral_completion
from tests.conftest import build_test_vibe_config
from vibe.core.tools.builtins.grep import GrepResult
from vibe.core.tools.builtins.read import ReadResult
from vibe.core.tools.builtins.todo import TodoResult
@ -33,7 +34,7 @@ async def _run_tool(
# Drive one tool call against the real cwd, then a final reply, and return
# the single tool result for the test to assert on.
mistral_api.reply(_tool_call(name, arguments), mistral_completion("done"))
agent = build_e2e_agent_loop(config=e2e_config(enabled_tools=[name]))
agent = build_e2e_agent_loop(config=build_test_vibe_config(enabled_tools=[name]))
events: list[BaseEvent] = [event async for event in agent.act("go")]
result = next(e for e in events if isinstance(e, ToolResultEvent))
assert result.error is None