uvos/vibe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2.18.0 (#843)

Browse source 
Co-authored-by: Clément Drouin <clement.drouin@mistral.ai>
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Cyprien <courtot.c@gmail.com>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com>
Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Nelson PROIA <144663685+Nelson-PROIA@users.noreply.github.com>
Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Quentin <quentin.torroba@mistral.ai>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: josephine-delas <57808586+josephine-delas@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Laure Hugo 2026-06-25 16:08:45 +02:00 • committed by GitHub
parent 725d3a56ce
commit e607ccbb00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
242 changed files with 7372 additions and 1974 deletions
Download patch file Download diff file Expand all files Collapse all files

271
tests/agent_loop/e2e/test_e2e_provider_apis.py
View file

@ -1,135 +1,146 @@
from __future__ import annotations
import pytest
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from tests.agent_loop.e2e.conftest import (
import pytest
import respx
from tests.agent_loop.e2e.conftest import build_e2e_agent_loop
from tests.agent_loop.e2e.providers import (
anthropic,
openai_responses,
reasoning,
vertex,
)
from tests.agent_loop.e2e.providers.base import (
ProviderAPI,
anthropic_e2e_config,
ProviderMocks,
assistant_text,
build_e2e_agent_loop,
openai_responses_e2e_config,
)
from tests.backend.data.anthropic import (
anthropic_message,
anthropic_reasoning_tool_use_stream,
anthropic_request_content_blocks,
anthropic_text_stream,
anthropic_tool_use,
)
from tests.backend.data.openai_responses import (
openai_function_call_item,
openai_message_item,
openai_reasoning_tool_call_stream,
openai_response,
openai_text_stream,
)
from tests.backend.data import ANSWER_CONTEXT_TOKENS
from tests.backend.data.anthropic import anthropic_message
from tests.backend.data.reasoning import reasoning_thinking_message
from vibe.core.config import VibeConfig
from vibe.core.types import ToolResultEvent
class TestAnthropic:
@dataclass(frozen=True)
class ProviderScenario:
"""Everything the shared e2e suite needs to exercise one provider.
Adding a provider is: implement `ProviderMocks`, then append one entry here.
"""
id: str
config: Callable[..., VibeConfig]
api_cls: type[ProviderAPI]
mocks: ProviderMocks
PROVIDER_SCENARIOS: list[ProviderScenario] = [
ProviderScenario(
id="anthropic",
config=anthropic.e2e_config,
api_cls=anthropic.API,
mocks=anthropic.Mocks(),
),
ProviderScenario(
id="openai-responses",
config=openai_responses.e2e_config,
api_cls=openai_responses.API,
mocks=openai_responses.Mocks(),
),
ProviderScenario(
id="reasoning",
config=reasoning.e2e_config,
api_cls=reasoning.API,
mocks=reasoning.Mocks(),
),
ProviderScenario(
id="vertex-anthropic",
config=vertex.e2e_config,
api_cls=vertex.API,
mocks=vertex.Mocks(),
),
]
@contextmanager
def open_provider_api(
api_cls: type[ProviderAPI], request: pytest.FixtureRequest
) -> Iterator[ProviderAPI]:
"""Instantiate a provider API and setup respx routers and monkeypatches"""
api_cls.setup_monkeypatch(request.getfixturevalue("monkeypatch"))
with respx.mock(base_url=api_cls.base_url, assert_all_called=False) as router:
api = api_cls()
api.setup_router(router)
yield api
@pytest.fixture(params=PROVIDER_SCENARIOS, ids=lambda s: s.id)
def scenario(request: pytest.FixtureRequest) -> ProviderScenario:
"""Parametrized fixture for each provider scenario in the shared e2e suite."""
return request.param
@pytest.fixture
def provider_api(
scenario: ProviderScenario, request: pytest.FixtureRequest
) -> Iterator[ProviderAPI]:
"""Returned setup provider API for the current scenario"""
with open_provider_api(scenario.api_cls, request) as api:
yield api
class TestProviderCommonBehaviors:
"""Answer / stream / tool-call behaviors every provider shares."""
@pytest.mark.asyncio
async def test_agent_answers(self, anthropic_api: ProviderAPI) -> None:
# A plain prompt is answered through the Anthropic messages wire.
anthropic_api.reply(anthropic_message("pong"))
agent = build_e2e_agent_loop(config=anthropic_e2e_config())
async def test_agent_answers(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
provider_api.reply(scenario.mocks.answer("pong"))
agent = build_e2e_agent_loop(config=scenario.config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 15
assert agent.stats.context_tokens == ANSWER_CONTEXT_TOKENS
@pytest.mark.asyncio
async def test_agent_streams(self, anthropic_api: ProviderAPI) -> None:
# Anthropic SSE deltas are reassembled into the final assistant content.
anthropic_api.reply_stream(anthropic_text_stream("pong"))
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(), enable_streaming=True
)
async def test_agent_streams(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
provider_api.reply_stream(scenario.mocks.text_stream("pong"))
agent = build_e2e_agent_loop(config=scenario.config(), enable_streaming=True)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_executes_tool_call(self, anthropic_api: ProviderAPI) -> None:
# A tool_use turn runs the tool, then Anthropic returns the final answer.
anthropic_api.reply(
anthropic_tool_use("todo", {"action": "read"}),
anthropic_message("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"])
async def test_agent_executes_tool_call(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
provider_api.reply(
scenario.mocks.tool_call("todo", {"action": "read"}),
scenario.mocks.answer("Your list is empty."),
)
agent = build_e2e_agent_loop(config=scenario.config(enabled_tools=["todo"]))
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, anthropic_api: ProviderAPI
) -> None:
# A streamed thinking + tool_use turn runs the tool, then replays that
# reasoning/tool history into the follow-up Anthropic request.
anthropic_api.reply_streams(
anthropic_reasoning_tool_use_stream("todo", '{"action": "read"}'),
anthropic_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=anthropic_e2e_config(enabled_tools=["todo"]), enable_streaming=True
)
events = [event async for event in agent.act("What's on my todo list?")]
assert "Your list is empty." in assistant_text(events)
blocks = anthropic_request_content_blocks(anthropic_api.request_json)
thinking = [b["thinking"] for b in blocks if b.get("type") == "thinking"]
tool_uses = [b for b in blocks if b.get("type") == "tool_use"]
assert any("thinking..." in text for text in thinking)
assert tool_uses
class TestOpenAIResponses:
@pytest.mark.asyncio
async def test_agent_answers(self, openai_responses_api: ProviderAPI) -> None:
# A plain prompt is answered through the OpenAI Responses wire.
openai_responses_api.reply(openai_response([openai_message_item("pong")]))
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert agent.stats.context_tokens == 12
@pytest.mark.asyncio
async def test_agent_streams(self, openai_responses_api: ProviderAPI) -> None:
# OpenAI Responses SSE deltas are reassembled into the final content.
openai_responses_api.reply_stream(openai_text_stream("pong"))
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(), enable_streaming=True
)
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
@pytest.mark.asyncio
async def test_agent_captures_reasoning(
self, openai_responses_api: ProviderAPI
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
# Commentary phase output is captured as reasoning on the assistant message.
openai_responses_api.reply(
openai_response(
[
openai_message_item("Let me think.", phase="commentary"),
openai_message_item("pong", phase="final_answer"),
],
output_tokens=5,
)
)
agent = build_e2e_agent_loop(config=openai_responses_e2e_config())
provider_api.reply(scenario.mocks.reasoning_answer("pong", "Let me think."))
agent = build_e2e_agent_loop(config=scenario.config())
events = [event async for event in agent.act("Reply with exactly: pong")]
@ -140,39 +151,55 @@ class TestOpenAIResponses:
)
@pytest.mark.asyncio
async def test_agent_executes_tool_call(
self, openai_responses_api: ProviderAPI
async def test_agent_streams_reasoning_then_runs_tool(
self, scenario: ProviderScenario, provider_api: ProviderAPI
) -> None:
# A function_call item runs the tool, then OpenAI returns the final answer.
openai_responses_api.reply(
openai_response([openai_function_call_item("todo", '{"action": "read"}')]),
openai_response([openai_message_item("Your list is empty.")]),
provider_api.reply_streams(
scenario.mocks.reasoning_tool_call_stream(
"todo", {"action": "read"}, reasoning="thinking..."
),
scenario.mocks.text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"])
config=scenario.config(enabled_tools=["todo"]), enable_streaming=True
)
events = [event async for event in agent.act("What's on my todo list?")]
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
assert any(
m.reasoning_content and "thinking..." in m.reasoning_content
for m in agent.messages
)
# The following tests are provider-specific and not shared across all providers, so they are not parametrized by scenario.
class TestReasoning:
@pytest.mark.asyncio
async def test_agent_streams_reasoning_and_tool_call(
self, openai_responses_api: ProviderAPI
async def test_forwards_thinking_level_as_reasoning_effort(
self, request: pytest.FixtureRequest
) -> None:
# A streamed commentary + function_call turn runs the tool, then OpenAI
# streams the final answer on the follow-up request.
openai_responses_api.reply_streams(
openai_reasoning_tool_call_stream("todo", '{"action": "read"}'),
openai_text_stream("Your list is empty."),
)
agent = build_e2e_agent_loop(
config=openai_responses_e2e_config(enabled_tools=["todo"]),
enable_streaming=True,
)
# The model's thinking level maps to reasoning_effort on the wire.
with open_provider_api(reasoning.API, request) as api:
api.reply(reasoning_thinking_message("pong", "Let me think."))
agent = build_e2e_agent_loop(config=reasoning.e2e_config(thinking="medium"))
events = [event async for event in agent.act("What's on my todo list?")]
async for _ in agent.act("Reply with exactly: pong"):
pass
assert any(isinstance(e, ToolResultEvent) for e in events)
assert "Your list is empty." in assistant_text(events)
assert api.request_json["reasoning_effort"] == "medium"
class TestVertexAnthropic:
@pytest.mark.asyncio
async def test_uses_vertex_wire(self, request: pytest.FixtureRequest) -> None:
# The request goes out on the Vertex rawPredict wire format.
with open_provider_api(vertex.API, request) as api:
api.reply(anthropic_message("pong"))
agent = build_e2e_agent_loop(config=vertex.e2e_config())
events = [event async for event in agent.act("Reply with exactly: pong")]
assert assistant_text(events) == "pong"
assert api.request_json["anthropic_version"] == "vertex-2023-10-16"