v2.3.0 (#429)

Co-authored-by: Carlo <carloantonio.patti@mistral.ai>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Clement Sirieix <clem.sirieix@gmail.com>
Co-authored-by: Michel Thomazo <michel.thomazo@mistral.ai>
Co-authored-by: Clément Drouin <clement.drouin@mistral.ai>
Co-authored-by: Vincent Guilloux <vincent.guilloux@mistral.ai>
Co-authored-by: Thomas Kenbeek <thomas.kenbeek@mistral.ai>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>

tests/e2e/common.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from contextlib import AbstractContextManager
+import io
+from pathlib import Path
+import re
+import time
+from typing import Protocol
+
+import pexpect
+
+
+class SpawnedVibeProcessFixture(Protocol):
+    def __call__(
+        self, workdir: Path
+    ) -> AbstractContextManager[tuple[pexpect.spawn, io.StringIO]]: ...
+
+
+def ansi_tolerant_pattern(text: str) -> re.Pattern[str]:
+    ansi = r"(?:\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07]*\x07|\r|\n)*"
+    return re.compile(ansi.join(re.escape(char) for char in text))
+
+
+def write_e2e_config(vibe_home: Path, api_base: str) -> None:
+    vibe_home.mkdir(parents=True, exist_ok=True)
+    (vibe_home / "config.toml").write_text(
+        "\n".join([
+            'active_model = "mock-model"',
+            "enable_update_checks = false",
+            "enable_auto_update = false",
+            "",
+            "[[providers]]",
+            'name = "mock-provider"',
+            f'api_base = "{api_base}"',
+            'api_key_env_var = "MISTRAL_API_KEY"',
+            'backend = "generic"',
+            "",
+            "[[models]]",
+            'name = "mock-model"',
+            'provider = "mock-provider"',
+            'alias = "mock-model"',
+        ]),
+        encoding="utf-8",
+    )
+
+
+def strip_ansi(text: str) -> str:
+    return re.sub(r"\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07]*\x07", "", text)
+
+
+def wait_for_request_count(
+    request_count_getter: Callable[[], int], expected_count: int, timeout: float
+) -> None:
+    start = time.monotonic()
+    while time.monotonic() - start < timeout:
+        if request_count_getter() >= expected_count:
+            return
+        time.sleep(0.05)
+    raise AssertionError(f"Timed out waiting for {expected_count} backend request(s).")
+
+
+def wait_for_main_screen(child: pexpect.spawn, timeout: float = 20.0) -> None:
+    child.expect(ansi_tolerant_pattern("Mistral Vibe v"), timeout=timeout)
+
+
+def wait_for_rendered_text(
+    child: pexpect.spawn, captured: io.StringIO, needle: str, timeout: float
+) -> None:
+    start = time.monotonic()
+    while time.monotonic() - start < timeout:
+        if needle in strip_ansi(captured.getvalue()):
+            return
+        try:
+            child.expect(r"\S", timeout=0.1)
+        except pexpect.TIMEOUT:
+            pass
+        except pexpect.EOF as exc:
+            rendered_tail = strip_ansi(captured.getvalue())[-1200:]
+            raise AssertionError(
+                f"Child exited while waiting for rendered text: {needle!r}\n\nRendered tail:\n{rendered_tail}"
+            ) from exc
+    rendered_tail = strip_ansi(captured.getvalue())[-1200:]
+    raise AssertionError(
+        f"Timed out waiting for rendered text: {needle!r}\n\nRendered tail:\n{rendered_tail}"
+    )

tests/e2e/conftest.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from contextlib import AbstractContextManager, contextmanager
+import io
+import os
+from pathlib import Path
+from typing import cast
+
+import pexpect
+import pytest
+
+from tests import TESTS_ROOT
+from tests.e2e.common import write_e2e_config
+from tests.e2e.mock_server import ChunkFactory, StreamingMockServer
+
+
+@pytest.fixture
+def streaming_mock_server(
+    request: pytest.FixtureRequest,
+) -> Iterator[StreamingMockServer]:
+    chunk_factory = cast(ChunkFactory | None, getattr(request, "param", None))
+    server = StreamingMockServer(chunk_factory=chunk_factory)
+    server.start()
+    try:
+        yield server
+    finally:
+        server.stop()
+
+
+@pytest.fixture
+def setup_e2e_env(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    streaming_mock_server: StreamingMockServer,
+) -> None:
+    vibe_home = tmp_path / "vibe-home"
+    write_e2e_config(vibe_home, streaming_mock_server.api_base)
+    monkeypatch.setenv("MISTRAL_API_KEY", "fake-key")
+    monkeypatch.setenv("VIBE_HOME", str(vibe_home))
+    monkeypatch.setenv("TERM", "xterm-256color")
+
+
+@pytest.fixture
+def e2e_workdir(tmp_path: Path) -> Path:
+    workdir = tmp_path / "workdir"
+    workdir.mkdir()
+    return workdir
+
+
+type SpawnedVibeContext = Iterator[tuple[pexpect.spawn, io.StringIO]]
+type SpawnedVibeContextManager = AbstractContextManager[
+    tuple[pexpect.spawn, io.StringIO]
+]
+type SpawnedVibeFactory = Callable[[Path], SpawnedVibeContextManager]
+
+
+@pytest.fixture
+def spawned_vibe_process() -> SpawnedVibeFactory:
+    @contextmanager
+    def spawn(workdir: Path) -> SpawnedVibeContext:
+        captured = io.StringIO()
+        child = pexpect.spawn(
+            "uv",
+            ["run", "vibe", "--workdir", str(workdir)],
+            cwd=str(TESTS_ROOT.parent),
+            env=os.environ,
+            encoding="utf-8",
+            timeout=30,
+            dimensions=(36, 120),
+        )
+        child.logfile_read = captured
+
+        try:
+            yield child, captured
+        finally:
+            if child.isalive():
+                child.terminate(force=True)
+            if not child.closed:
+                child.close()
+
+    return spawn

tests/e2e/mock_server.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import json
+import threading
+import time
+from typing import TypedDict, cast
+
+
+class StreamOptionsPayload(TypedDict, total=False):
+    include_usage: bool
+    stream_tool_calls: bool
+
+
+class ChatMessagePayload(TypedDict, total=False):
+    role: str
+    content: str
+
+
+class ChatCompletionsRequestPayload(TypedDict, total=False):
+    model: str
+    messages: list[ChatMessagePayload]
+    stream: bool
+    stream_options: StreamOptionsPayload
+
+
+type StreamChunk = dict[str, object]
+type ChunkFactory = Callable[[int, ChatCompletionsRequestPayload], list[StreamChunk]]
+
+
+class StreamingMockServer:
+    @staticmethod
+    def build_chunk(
+        *,
+        created: int,
+        delta: dict[str, object],
+        finish_reason: str | None,
+        usage: dict[str, int] | None = None,
+    ) -> StreamChunk:
+        chunk: dict[str, object] = {
+            "id": "mock-id",
+            "object": "chat.completion.chunk",
+            "created": created,
+            "model": "mock-model",
+            "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
+        }
+        if usage is not None:
+            chunk["usage"] = usage
+        return chunk
+
+    @staticmethod
+    def build_tool_call_delta(
+        *, call_id: str, tool_name: str, arguments: str, index: int = 0
+    ) -> dict[str, object]:
+        return {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "index": index,
+                    "id": call_id,
+                    "type": "function",
+                    "function": {"name": tool_name, "arguments": arguments},
+                }
+            ],
+        }
+
+    @staticmethod
+    def _stream_chunks() -> list[StreamChunk]:
+        return [
+            StreamingMockServer.build_chunk(
+                created=123,
+                delta={"role": "assistant", "content": "Hello"},
+                finish_reason=None,
+            ),
+            StreamingMockServer.build_chunk(
+                created=124, delta={"content": " from mock server"}, finish_reason=None
+            ),
+            StreamingMockServer.build_chunk(
+                created=125,
+                delta={},
+                finish_reason="stop",
+                usage={"prompt_tokens": 3, "completion_tokens": 4},
+            ),
+        ]
+
+    def __init__(self, *, chunk_factory: ChunkFactory | None = None) -> None:
+        self.requests: list[ChatCompletionsRequestPayload] = []
+        self._lock = threading.Lock()
+        self._chunk_factory = chunk_factory
+        self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._build_handler())
+        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+
+    def _build_handler(self) -> type[BaseHTTPRequestHandler]:
+        parent = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def log_message(self, format, *args):
+                return
+
+            def do_POST(self) -> None:
+                if self.path != "/v1/chat/completions":
+                    self.send_response(404)
+                    self.end_headers()
+                    return
+
+                length = int(self.headers.get("Content-Length", "0"))
+                body = self.rfile.read(length)
+                payload = cast(
+                    ChatCompletionsRequestPayload, json.loads(body.decode("utf-8"))
+                )
+
+                with parent._lock:
+                    parent.requests.append(payload)
+                    request_index = len(parent.requests) - 1
+
+                self.send_response(200)
+                self.send_header("Content-Type", "text/event-stream")
+                self.send_header("Cache-Control", "no-cache")
+                self.end_headers()
+
+                chunks = (
+                    parent._chunk_factory(request_index, payload)
+                    if parent._chunk_factory is not None
+                    else parent._stream_chunks()
+                )
+
+                for chunk in chunks:
+                    data = json.dumps(chunk, ensure_ascii=False)
+                    self.wfile.write(f"data: {data}\n\n".encode())
+                    self.wfile.flush()
+                    time.sleep(0.03)
+
+                self.wfile.write(b"data: [DONE]\n\n")
+                self.wfile.flush()
+
+        return Handler
+
+    @property
+    def api_base(self) -> str:
+        return f"http://127.0.0.1:{self._server.server_port}/v1"
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._server.shutdown()
+        self._server.server_close()
+        self._thread.join(timeout=1)

tests/e2e/test_cli_tui_onboarding.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pexpect
+import pytest
+
+from tests.e2e.common import SpawnedVibeProcessFixture, ansi_tolerant_pattern
+
+
+@pytest.mark.timeout(15)
+def test_spawn_cli_shows_onboarding_when_api_key_missing(
+    tmp_path: Path,
+    e2e_workdir: Path,
+    spawned_vibe_process: SpawnedVibeProcessFixture,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    vibe_home = tmp_path / "vibe-home-onboarding"
+    vibe_home.mkdir(parents=True, exist_ok=True)
+
+    monkeypatch.setenv("VIBE_HOME", str(vibe_home))
+    monkeypatch.setenv("TERM", "xterm-256color")
+    monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
+
+    with spawned_vibe_process(e2e_workdir) as (child, captured):
+        child.expect(ansi_tolerant_pattern("Welcome to Mistral Vibe"), timeout=15)
+        child.sendcontrol("c")
+        child.expect(pexpect.EOF, timeout=10)
+
+    output = captured.getvalue()
+    assert "Setup cancelled" in output

tests/e2e/test_cli_tui_streaming.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pexpect
+import pytest
+
+from tests.e2e.common import (
+    SpawnedVibeProcessFixture,
+    ansi_tolerant_pattern,
+    wait_for_main_screen,
+    wait_for_request_count,
+)
+from tests.e2e.mock_server import StreamingMockServer
+
+
+@pytest.mark.timeout(15)
+def test_spawn_cli_to_send_and_receive_message(
+    streaming_mock_server: StreamingMockServer,
+    setup_e2e_env: None,
+    e2e_workdir: Path,
+    spawned_vibe_process: SpawnedVibeProcessFixture,
+) -> None:
+    with spawned_vibe_process(e2e_workdir) as (child, captured):
+        wait_for_main_screen(child, timeout=15)
+        child.send("Greet")
+        child.send("\r")
+
+        wait_for_request_count(
+            lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
+        )
+        child.expect(ansi_tolerant_pattern("Hello from mock server"), timeout=10)
+
+        child.sendcontrol("c")
+        child.expect(pexpect.EOF, timeout=10)
+
+    output = captured.getvalue()
+    assert "Welcome to Mistral Vibe" not in output
+
+    request_payload = streaming_mock_server.requests[-1]
+    assert request_payload.get("stream") is True
+    assert request_payload.get("model") == "mock-model"
+    stream_options = request_payload.get("stream_options")
+    assert stream_options is not None
+    assert stream_options.get("include_usage") is True
+    messages = request_payload.get("messages")
+    assert messages is not None
+    assert any(
+        message.get("role") == "user" and message.get("content") == "Greet"
+        for message in messages
+    )

tests/e2e/test_cli_tui_tool_approval.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pexpect
+import pytest
+
+from tests.e2e.common import (
+    SpawnedVibeProcessFixture,
+    wait_for_main_screen,
+    wait_for_rendered_text,
+    wait_for_request_count,
+)
+from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
+
+PREDICTABLE_OUTPUT = "__E2E_BASH_OK__"
+TOOL_ARGUMENTS = f'{{"command":"printf \\"{PREDICTABLE_OUTPUT}\\\\n\\""}}'
+
+
+def _tool_call_factory(
+    request_index: int, _payload: ChatCompletionsRequestPayload
+) -> list[dict[str, object]]:
+    if request_index == 0:
+        return [
+            StreamingMockServer.build_chunk(
+                created=1,
+                delta=StreamingMockServer.build_tool_call_delta(
+                    call_id="call_bash_1", tool_name="bash", arguments=TOOL_ARGUMENTS
+                ),
+                finish_reason=None,
+            ),
+            StreamingMockServer.build_chunk(
+                created=2,
+                delta={},
+                finish_reason="tool_calls",
+                usage={"prompt_tokens": 3, "completion_tokens": 4},
+            ),
+        ]
+
+    return [
+        StreamingMockServer.build_chunk(
+            created=3,
+            delta={
+                "role": "assistant",
+                "content": f"The string {PREDICTABLE_OUTPUT} has been printed successfully.",
+            },
+            finish_reason=None,
+        ),
+        StreamingMockServer.build_chunk(
+            created=4, delta={"content": PREDICTABLE_OUTPUT}, finish_reason=None
+        ),
+        StreamingMockServer.build_chunk(
+            created=5,
+            delta={},
+            finish_reason="stop",
+            usage={"prompt_tokens": 3, "completion_tokens": 4},
+        ),
+    ]
+
+
+@pytest.mark.timeout(25)
+@pytest.mark.parametrize(
+    "streaming_mock_server",
+    [pytest.param(_tool_call_factory, id="tool-call-stream")],
+    indirect=True,
+)
+def test_spawn_cli_asks_bash_permission_and_shows_tool_output_after_approval(
+    streaming_mock_server: StreamingMockServer,
+    setup_e2e_env: None,
+    e2e_workdir: Path,
+    spawned_vibe_process: SpawnedVibeProcessFixture,
+) -> None:
+    with spawned_vibe_process(e2e_workdir) as (child, captured):
+        wait_for_main_screen(child, timeout=15)
+        child.send("Run a shell command")
+        child.send("\r")
+
+        wait_for_request_count(
+            lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
+        )
+        wait_for_rendered_text(child, captured, needle="bash command", timeout=10)
+        child.send("y")
+        child.send("\r")
+        wait_for_rendered_text(child, captured, needle=PREDICTABLE_OUTPUT, timeout=10)
+
+        child.sendcontrol("c")
+        child.expect(pexpect.EOF, timeout=10)