v2.18.0 (#843)

Co-authored-by: Clément Drouin <clement.drouin@mistral.ai>
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Cyprien <courtot.c@gmail.com>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com>
Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Nelson PROIA <144663685+Nelson-PROIA@users.noreply.github.com>
Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Quentin <quentin.torroba@mistral.ai>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: josephine-delas <57808586+josephine-delas@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>

tests/agent_loop/e2e/providers/__init__.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from tests.agent_loop.e2e.providers.base import (
+    ProviderAPI,
+    ProviderMocks,
+    assistant_text,
+)
+from tests.agent_loop.e2e.providers.mistral import MistralAPI
+
+__all__ = ["MistralAPI", "ProviderAPI", "ProviderMocks", "assistant_text"]

tests/agent_loop/e2e/providers/anthropic.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from tests import constants as c
+from tests.agent_loop.e2e.providers.base import ProviderAPI
+from tests.backend.data import Chunk, JsonResponse
+from tests.backend.data.anthropic import (
+    anthropic_message,
+    anthropic_reasoning_tool_use_stream,
+    anthropic_text_stream,
+    anthropic_tool_use,
+)
+from tests.conftest import build_test_vibe_config
+from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
+from vibe.core.types import Backend
+
+
+def e2e_config(**overrides: Any) -> VibeConfig:
+    provider = ProviderConfig(
+        name="anthropic",
+        api_base=c.ANTHROPIC_BASE_URL,
+        api_key_env_var="ANTHROPIC_API_KEY",
+        api_style="anthropic",
+        backend=Backend.GENERIC,
+    )
+    models = [ModelConfig(name="claude-test", provider="anthropic", alias="anthropic")]
+    return build_test_vibe_config(
+        active_model="anthropic", models=models, providers=[provider], **overrides
+    )
+
+
+class API(ProviderAPI):
+    base_url = c.ANTHROPIC_BASE_URL
+    post_path = c.ANTHROPIC_MESSAGES_PATH
+
+
+class Mocks:
+    def answer(self, text: str) -> JsonResponse:
+        return anthropic_message(text)
+
+    def text_stream(self, text: str) -> list[Chunk]:
+        return anthropic_text_stream(text)
+
+    def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
+        return anthropic_tool_use(name, arguments)
+
+    def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
+        message = anthropic_message(text)
+        message["content"].insert(
+            0, {"type": "thinking", "thinking": reasoning, "signature": "sig"}
+        )
+        return message
+
+    def reasoning_tool_call_stream(
+        self, name: str, arguments: dict[str, Any], *, reasoning: str
+    ) -> list[Chunk]:
+        return anthropic_reasoning_tool_use_stream(
+            name, json.dumps(arguments), reasoning=reasoning
+        )

tests/agent_loop/e2e/providers/base.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+from typing import Any, ClassVar, Protocol
+
+import httpx
+import pytest
+import respx
+
+from tests.backend.data import Chunk, JsonResponse
+from vibe.core.types import AssistantEvent, BaseEvent
+
+
+def assistant_text(events: list[BaseEvent]) -> str:
+    return "".join(
+        e.content for e in events if isinstance(e, AssistantEvent) and e.content
+    ).strip()
+
+
+class ProviderMocks(Protocol):
+    """Canonical wire payloads a provider must produce for the shared e2e suite."""
+
+    def answer(self, text: str) -> JsonResponse:
+        """A non-streaming assistant turn that replies with `text`."""
+        ...
+
+    def text_stream(self, text: str) -> list[Chunk]:
+        """A streaming assistant turn whose deltas reassemble into `text`."""
+        ...
+
+    def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
+        """A non-streaming turn that calls tool `name` with `arguments`."""
+        ...
+
+    def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
+        """A non-streaming turn that thinks `reasoning`, then replies `text`."""
+        ...
+
+    def reasoning_tool_call_stream(
+        self, name: str, arguments: dict[str, Any], *, reasoning: str
+    ) -> list[Chunk]:
+        """A streamed turn that thinks `reasoning`, then calls tool `name`."""
+        ...
+
+
+class ProviderAPI:
+    """Stubs a provider's completion wire; the AgentLoop stays the subject.
+
+    Subclasses describe one provider via `base_url`/`post_path`; override
+    `setup_monkeypatch` for non-HTTP stubs (e.g. credentials) a provider needs.
+    """
+
+    base_url: ClassVar[str]
+    post_path: ClassVar[str]
+
+    route: respx.Route
+
+    def setup_router(self, router: respx.MockRouter) -> None:
+        """Bind the completion route; subclasses extend for extra wires/defaults."""
+        self.route = router.post(self.post_path)
+
+    @classmethod
+    def setup_monkeypatch(cls, monkeypatch: pytest.MonkeyPatch) -> None:
+        """Apply non-HTTP stubs (e.g. credential lookups) before use."""
+
+    def reply(self, *completions: dict[str, Any]) -> None:
+        responses = [httpx.Response(200, json=completion) for completion in completions]
+        if len(responses) == 1:
+            self.route.mock(return_value=responses[0])
+        else:
+            self.route.mock(side_effect=responses)
+
+    def reply_stream(self, chunks: list[bytes]) -> None:
+        self.route.mock(return_value=self._stream_response(chunks))
+
+    def reply_streams(self, *chunk_lists: list[bytes]) -> None:
+        self.route.mock(
+            side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
+        )
+
+    @staticmethod
+    def _stream_response(chunks: list[bytes]) -> httpx.Response:
+        return httpx.Response(
+            200,
+            stream=httpx.ByteStream(stream=b"\n\n".join(chunks)),
+            headers={"Content-Type": "text/event-stream"},
+        )
+
+    @property
+    def request_json(self) -> dict[str, Any]:
+        return json.loads(self.route.calls.last.request.content)
+
+    def model_facing_text(self, index: int) -> str:
+        """Return the text (raw prompt) that the model would have seen in the request
+        at the given call index.
+        """
+        body = json.loads(self.route.calls[index].request.content)
+        return json.dumps([m for m in body["messages"] if m.get("role") != "system"])

tests/agent_loop/e2e/providers/mistral.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import httpx
+import respx
+
+from tests import constants as c
+from tests.agent_loop.e2e.providers.base import ProviderAPI
+
+
+class MistralAPI(ProviderAPI):
+    base_url = c.MISTRAL_BASE_URL
+    post_path = c.CHAT_COMPLETIONS_PATH
+
+    def setup_router(self, router: respx.MockRouter) -> None:
+        super().setup_router(router)
+        router.get(c.CONNECTORS_BOOTSTRAP_PATH).mock(
+            return_value=httpx.Response(200, json={"connectors": []})
+        )

tests/agent_loop/e2e/providers/openai_responses.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from tests import constants as c
+from tests.agent_loop.e2e.providers.base import ProviderAPI
+from tests.backend.data import Chunk, JsonResponse
+from tests.backend.data.openai_responses import (
+    openai_function_call_item,
+    openai_message_item,
+    openai_reasoning_tool_call_stream,
+    openai_response,
+    openai_text_stream,
+)
+from tests.conftest import build_test_vibe_config
+from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
+from vibe.core.types import Backend
+
+
+def e2e_config(**overrides: Any) -> VibeConfig:
+    provider = ProviderConfig(
+        name="openai",
+        api_base=f"{c.OPENAI_BASE_URL}/v1",
+        api_key_env_var="OPENAI_API_KEY",
+        api_style="openai-responses",
+        backend=Backend.GENERIC,
+    )
+    models = [ModelConfig(name="gpt-test", provider="openai", alias="openai")]
+    return build_test_vibe_config(
+        active_model="openai", models=models, providers=[provider], **overrides
+    )
+
+
+class API(ProviderAPI):
+    base_url = c.OPENAI_BASE_URL
+    post_path = c.OPENAI_RESPONSES_PATH
+
+
+class Mocks:
+    def answer(self, text: str) -> JsonResponse:
+        return openai_response([openai_message_item(text)])
+
+    def text_stream(self, text: str) -> list[Chunk]:
+        return openai_text_stream(text)
+
+    def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
+        return openai_response([openai_function_call_item(name, json.dumps(arguments))])
+
+    def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
+        return openai_response([
+            openai_message_item(reasoning, phase="commentary"),
+            openai_message_item(text, phase="final_answer"),
+        ])
+
+    def reasoning_tool_call_stream(
+        self, name: str, arguments: dict[str, Any], *, reasoning: str
+    ) -> list[Chunk]:
+        return openai_reasoning_tool_call_stream(
+            name, json.dumps(arguments), reasoning=reasoning
+        )

tests/agent_loop/e2e/providers/reasoning.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from tests import constants as c
+from tests.agent_loop.e2e.providers.base import ProviderAPI
+from tests.backend.data import Chunk, JsonResponse
+from tests.backend.data.reasoning import (
+    reasoning_message,
+    reasoning_text_stream,
+    reasoning_thinking_message,
+    reasoning_thinking_tool_use_stream,
+    reasoning_tool_use,
+)
+from tests.conftest import build_test_vibe_config
+from vibe.core.config import ModelConfig, ProviderConfig, ThinkingLevel, VibeConfig
+from vibe.core.types import Backend
+
+
+def e2e_config(*, thinking: ThinkingLevel = "off", **overrides: Any) -> VibeConfig:
+    provider = ProviderConfig(
+        name="reasoning",
+        api_base=c.REASONING_BASE_URL,
+        api_key_env_var="REASONING_API_KEY",
+        api_style="reasoning",
+        backend=Backend.GENERIC,
+    )
+    models = [
+        ModelConfig(
+            name="reasoning-test",
+            provider="reasoning",
+            alias="reasoning",
+            thinking=thinking,
+        )
+    ]
+    return build_test_vibe_config(
+        active_model="reasoning", models=models, providers=[provider], **overrides
+    )
+
+
+class API(ProviderAPI):
+    base_url = c.REASONING_BASE_URL
+    post_path = c.REASONING_COMPLETIONS_PATH
+
+
+class Mocks:
+    def answer(self, text: str) -> JsonResponse:
+        return reasoning_message(text)
+
+    def text_stream(self, text: str) -> list[Chunk]:
+        return reasoning_text_stream(text)
+
+    def tool_call(self, name: str, arguments: dict[str, Any]) -> JsonResponse:
+        return reasoning_tool_use(name, json.dumps(arguments))
+
+    def reasoning_answer(self, text: str, reasoning: str) -> JsonResponse:
+        return reasoning_thinking_message(text, reasoning)
+
+    def reasoning_tool_call_stream(
+        self, name: str, arguments: dict[str, Any], *, reasoning: str
+    ) -> list[Chunk]:
+        return reasoning_thinking_tool_use_stream(
+            name, json.dumps(arguments), reasoning=reasoning
+        )

tests/agent_loop/e2e/providers/vertex.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+import respx
+
+from tests import constants as c
+from tests.agent_loop.e2e.providers.anthropic import Mocks
+from tests.agent_loop.e2e.providers.base import ProviderAPI
+from tests.conftest import build_test_vibe_config
+from vibe.core.config import ModelConfig, ProviderConfig, VibeConfig
+from vibe.core.types import Backend
+
+__all__ = ["API", "Mocks", "e2e_config"]
+
+
+def e2e_config(**overrides: Any) -> VibeConfig:
+    provider = ProviderConfig(
+        name="vertex",
+        api_base="",
+        api_key_env_var="VERTEX_API_KEY",
+        api_style="vertex-anthropic",
+        backend=Backend.GENERIC,
+        project_id=c.VERTEX_PROJECT_ID,
+        region=c.VERTEX_REGION,
+    )
+    models = [ModelConfig(name=c.VERTEX_MODEL, provider="vertex", alias="vertex")]
+    return build_test_vibe_config(
+        active_model="vertex", models=models, providers=[provider], **overrides
+    )
+
+
+class API(ProviderAPI):
+    """Vertex splits rawPredict and streamRawPredict across two wire paths."""
+
+    base_url = c.VERTEX_BASE_URL
+    post_path = c.VERTEX_RAW_PREDICT_PATH
+
+    def setup_router(self, router: respx.MockRouter) -> None:
+        super().setup_router(router)
+        self._stream_route = router.post(c.VERTEX_STREAM_PREDICT_PATH)
+
+    @classmethod
+    def setup_monkeypatch(cls, monkeypatch: pytest.MonkeyPatch) -> None:
+        # google.auth is the only external dependency Vertex pulls in; stub the
+        # credential lookup so the real token-refresh logic runs without network.
+        creds = MagicMock()
+        creds.valid = True
+        creds.token = "fake-vertex-token"
+        monkeypatch.setattr(
+            "vibe.core.llm.backend.vertex.google.auth.default",
+            lambda **_kwargs: (creds, c.VERTEX_PROJECT_ID),
+        )
+
+    def reply_stream(self, chunks: list[bytes]) -> None:
+        self._stream_route.mock(return_value=self._stream_response(chunks))
+
+    def reply_streams(self, *chunk_lists: list[bytes]) -> None:
+        self._stream_route.mock(
+            side_effect=[self._stream_response(chunks) for chunks in chunk_lists]
+        )
+
+    @property
+    def request_json(self) -> dict[str, Any]:
+        route = self._stream_route if self._stream_route.calls else self.route
+        return json.loads(route.calls.last.request.content)