uvos/vibe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2.17.0 (#822)

Browse source 
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com>
Co-authored-by: Hdandria <henri.dandria@mistral.ai>
Co-authored-by: Ivana Dunisijevic <ivana.dunisijevic@mistral.ai>
Co-authored-by: Jean Burellier <sheplu@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Mert Unsal <mert.unsal@mistral.ai>
Co-authored-by: Michel Thomazo <51709227+michelTho@users.noreply.github.com>
Co-authored-by: Paul VEZIA <166131032+le-codeur-rapide@users.noreply.github.com>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Val <102326092+vdeva@users.noreply.github.com>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: renovate-mistral[bot] <253709520+renovate-mistral[bot]@users.noreply.github.com>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Clément Drouin 2026-06-19 11:01:24 +02:00 • committed by GitHub
parent 564a14365e
commit 6bedf271ce
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
223 changed files with 10533 additions and 6947 deletions
Download patch file Download diff file Expand all files Collapse all files

178
tests/e2e/agent_loop_characterization/support.py Normal file
View file

@ -0,0 +1,178 @@
from __future__ import annotations
from collections.abc import Callable
import io
import json
from pathlib import Path
import time
import tomllib
from typing import Any
import pexpect
import tomli_w
from tests.e2e.common import strip_ansi, wait_for_rendered_text
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
APPROVAL_INPUT_GRACE_PERIOD_S = 0.65
def assistant_text_chunks(text: str, *, created: int = 100) -> list[dict[str, object]]:
return [
StreamingMockServer.build_chunk(
created=created,
delta={"role": "assistant", "content": text},
finish_reason=None,
),
StreamingMockServer.build_chunk(
created=created + 1,
delta={},
finish_reason="stop",
usage={"prompt_tokens": 3, "completion_tokens": 4},
),
]
def single_tool_call_chunks(
*, call_id: str, tool_name: str, arguments: dict[str, Any], created: int = 10
) -> list[dict[str, object]]:
return [
StreamingMockServer.build_chunk(
created=created,
delta=StreamingMockServer.build_tool_call_delta(
call_id=call_id,
tool_name=tool_name,
arguments=json.dumps(arguments, separators=(",", ":")),
),
finish_reason=None,
),
StreamingMockServer.build_chunk(
created=created + 1,
delta={},
finish_reason="tool_calls",
usage={"prompt_tokens": 3, "completion_tokens": 4},
),
]
def multi_tool_call_chunks(
tool_calls: list[tuple[str, str, dict[str, Any]]], *, created: int = 10
) -> list[dict[str, object]]:
return [
StreamingMockServer.build_chunk(
created=created,
delta={
"role": "assistant",
"tool_calls": [
{
"index": index,
"id": call_id,
"type": "function",
"function": {
"name": tool_name,
"arguments": json.dumps(arguments, separators=(",", ":")),
},
}
for index, (call_id, tool_name, arguments) in enumerate(tool_calls)
],
},
finish_reason=None,
),
StreamingMockServer.build_chunk(
created=created + 1,
delta={},
finish_reason="tool_calls",
usage={"prompt_tokens": 3, "completion_tokens": 4},
),
]
def _messages(payload: ChatCompletionsRequestPayload) -> list[dict[str, Any]]:
raw_messages = payload.get("messages")
assert raw_messages is not None
return [dict(message) for message in raw_messages]
def assert_tool_result_contains(
payload: ChatCompletionsRequestPayload, *, call_id: str, expected: str
) -> None:
matching_messages = [
message
for message in _messages(payload)
if message.get("role") == "tool" and message.get("tool_call_id") == call_id
]
assert len(matching_messages) == 1
content = matching_messages[0].get("content")
assert isinstance(content, str)
assert expected in content, content
def assert_assistant_tool_call_present(
payload: ChatCompletionsRequestPayload, *, call_id: str, tool_name: str
) -> None:
for message in _messages(payload):
if message.get("role") != "assistant":
continue
tool_calls = message.get("tool_calls")
if not isinstance(tool_calls, list):
continue
for tool_call in tool_calls:
if not isinstance(tool_call, dict):
continue
function = tool_call.get("function")
if not isinstance(function, dict):
continue
if tool_call.get("id") == call_id and function.get("name") == tool_name:
return
raise AssertionError(f"Tool call {call_id!r} for {tool_name!r} was not present.")
def assert_message_content_present(
payload: ChatCompletionsRequestPayload, *, role: str, expected: str
) -> None:
assert any(
message.get("role") == role and expected in str(message.get("content", ""))
for message in _messages(payload)
)
def wait_for_request_count_while_draining_child_output(
child: pexpect.spawn,
captured: io.StringIO,
request_count_getter: Callable[[], int],
*,
expected_count: int,
timeout: float,
) -> None:
start = time.monotonic()
while time.monotonic() - start < timeout:
if request_count_getter() >= expected_count:
return
try:
child.expect(r"\S", timeout=0.05)
except pexpect.TIMEOUT:
pass
rendered_tail = strip_ansi(captured.getvalue())[-1200:]
raise AssertionError(
f"Timed out waiting for {expected_count} backend request(s).\n\n"
f"Rendered tail:\n{rendered_tail}"
)
def answer_approval(
child: pexpect.spawn, captured: io.StringIO, *, tool_name: str, key: str
) -> None:
wait_for_rendered_text(
child, captured, needle=f"Permission for the {tool_name} tool", timeout=10
)
wait_for_rendered_text(child, captured, needle="Deny", timeout=10)
time.sleep(APPROVAL_INPUT_GRACE_PERIOD_S)
child.send(key)
child.send("\r")
def set_tool_denylist(vibe_home: Path, tool_name: str, patterns: list[str]) -> None:
config_path = vibe_home / "config.toml"
config = tomllib.loads(config_path.read_text(encoding="utf-8"))
config.setdefault("tools", {}).setdefault(tool_name, {})["denylist"] = patterns
config_path.write_bytes(tomli_w.dumps(config).encode())

224
tests/e2e/agent_loop_characterization/test_resume.py Normal file
View file

@ -0,0 +1,224 @@
from __future__ import annotations
import json
import os
from pathlib import Path
import re
import time
import pexpect
import pytest
from tests.e2e.agent_loop_characterization.support import (
assert_assistant_tool_call_present,
assert_message_content_present,
assert_tool_result_contains,
assistant_text_chunks,
single_tool_call_chunks,
wait_for_request_count_while_draining_child_output,
)
from tests.e2e.common import (
SpawnedVibeProcessFixture,
send_ctrl_c_until_quit_confirmation,
strip_ansi,
wait_for_main_screen,
wait_for_rendered_text,
)
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
RESUME_TODO_CALL_ID = "call_todo_resume"
RESUME_INITIAL_PROMPT = "Start tool history"
RESUME_CONTINUE_PROMPT = "Continue from prior tool history"
RESUME_FIRST_TURN_RESPONSE = "First turn with todo complete."
RESUME_RESUMED_TURN_RESPONSE = "Resumed turn saw prior tool history."
RESUME_TODO_RESULT_TEXT = "Updated 1 todos"
def _load_latest_session_messages() -> list[dict[str, object]]:
session_root = Path(os.environ["VIBE_HOME"]) / "logs" / "session"
messages_paths = list(session_root.glob("session_*/messages.jsonl"))
if not messages_paths:
return []
messages_path = max(messages_paths, key=lambda path: path.stat().st_mtime)
messages: list[dict[str, object]] = []
try:
for line in messages_path.read_text(encoding="utf-8").splitlines():
messages.append(json.loads(line))
except (OSError, json.JSONDecodeError):
return []
return messages
def _has_message_content(
messages: list[dict[str, object]], *, role: str, expected: str
) -> bool:
return any(
message.get("role") == role and expected in str(message.get("content", ""))
for message in messages
)
def _has_assistant_tool_call(
messages: list[dict[str, object]], *, call_id: str
) -> bool:
return any(
message.get("role") == "assistant"
and call_id in json.dumps(message.get("tool_calls", []))
for message in messages
)
def _has_tool_result(
messages: list[dict[str, object]], *, call_id: str, expected: str
) -> bool:
return any(
message.get("role") == "tool"
and message.get("tool_call_id") == call_id
and expected in str(message.get("content", ""))
for message in messages
)
def _wait_for_persisted_resume_history(
*,
user_prompt: str,
assistant_tool_call_id: str,
tool_result_text: str,
final_assistant_text: str,
timeout: float,
) -> None:
start = time.monotonic()
while time.monotonic() - start < timeout:
messages = _load_latest_session_messages()
if (
_has_message_content(messages, role="user", expected=user_prompt)
and _has_assistant_tool_call(messages, call_id=assistant_tool_call_id)
and _has_tool_result(
messages, call_id=assistant_tool_call_id, expected=tool_result_text
)
and _has_message_content(
messages, role="assistant", expected=final_assistant_text
)
):
return
time.sleep(0.05)
persisted_roles = [
message.get("role") for message in _load_latest_session_messages()
]
raise AssertionError(
f"Timed out waiting for persisted resume history. Persisted roles: {persisted_roles}"
)
def _resume_tool_history_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return single_tool_call_chunks(
call_id=RESUME_TODO_CALL_ID,
tool_name="todo",
arguments={
"action": "write",
"todos": [
{
"id": "resume-history",
"content": "preserve tool history",
"status": "completed",
"priority": "high",
}
],
},
created=40,
)
if request_index == 1:
return assistant_text_chunks(RESUME_FIRST_TURN_RESPONSE, created=50)
return assistant_text_chunks(RESUME_RESUMED_TURN_RESPONSE, created=60)
@pytest.mark.timeout(35)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_resume_tool_history_factory, id="resume-tool-history")],
indirect=True,
)
def test_resumed_session_sends_prior_tool_call_and_result_history_to_the_model(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send(RESUME_INITIAL_PROMPT)
child.send("\r")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle=RESUME_FIRST_TURN_RESPONSE, timeout=10
)
_wait_for_persisted_resume_history(
user_prompt=RESUME_INITIAL_PROMPT,
assistant_tool_call_id=RESUME_TODO_CALL_ID,
tool_result_text=RESUME_TODO_RESULT_TEXT,
final_assistant_text=RESUME_FIRST_TURN_RESPONSE,
timeout=10,
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
first_output = strip_ansi(captured.getvalue())
resume_match = re.search(r"Or: vibe --resume ([0-9a-f-]+)", first_output)
assert resume_match is not None
session_id = resume_match.group(1)
with spawned_vibe_process(e2e_workdir, extra_args=["--resume", session_id]) as (
resumed_child,
resumed_captured,
):
wait_for_main_screen(resumed_child, timeout=15)
resumed_child.send(RESUME_CONTINUE_PROMPT)
resumed_child.send("\r")
wait_for_request_count_while_draining_child_output(
resumed_child,
resumed_captured,
lambda: len(streaming_mock_server.requests),
expected_count=3,
timeout=10,
)
wait_for_rendered_text(
resumed_child,
resumed_captured,
needle=RESUME_RESUMED_TURN_RESPONSE,
timeout=10,
)
send_ctrl_c_until_quit_confirmation(resumed_child, resumed_captured, timeout=5)
resumed_child.expect(pexpect.EOF, timeout=10)
resumed_payload = streaming_mock_server.requests[2]
assert_message_content_present(
resumed_payload, role="user", expected=RESUME_INITIAL_PROMPT
)
assert_assistant_tool_call_present(
resumed_payload, call_id=RESUME_TODO_CALL_ID, tool_name="todo"
)
assert_tool_result_contains(
resumed_payload, call_id=RESUME_TODO_CALL_ID, expected=RESUME_TODO_RESULT_TEXT
)
assert_message_content_present(
resumed_payload, role="assistant", expected=RESUME_FIRST_TURN_RESPONSE
)
assert_message_content_present(
resumed_payload, role="user", expected=RESUME_CONTINUE_PROMPT
)

87
tests/e2e/agent_loop_characterization/test_subagents.py Normal file
View file

@ -0,0 +1,87 @@
from __future__ import annotations
from pathlib import Path
import pexpect
import pytest
from tests.e2e.agent_loop_characterization.support import (
assert_message_content_present,
assert_tool_result_contains,
assistant_text_chunks,
single_tool_call_chunks,
wait_for_request_count_while_draining_child_output,
)
from tests.e2e.common import (
SpawnedVibeProcessFixture,
send_ctrl_c_until_quit_confirmation,
wait_for_main_screen,
wait_for_rendered_text,
)
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
SUBAGENT_TOOL_CALL_ID = "call_subagent_explore"
SUBAGENT_MARKER = "__E2E_SUBAGENT_DONE__"
def _explore_subagent_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return single_tool_call_chunks(
call_id=SUBAGENT_TOOL_CALL_ID,
tool_name="task",
arguments={
"agent": "explore",
"task": f"Find the marker {SUBAGENT_MARKER}.",
},
created=90,
)
if request_index == 1:
return assistant_text_chunks(f"Subagent found {SUBAGENT_MARKER}.", created=100)
return assistant_text_chunks("Parent used the subagent result.", created=110)
@pytest.mark.timeout(30)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_explore_subagent_factory, id="explore-subagent")],
indirect=True,
)
def test_explore_subagent_returns_result_to_parent_model(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Delegate exploration")
child.send("\r")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=3,
timeout=15,
)
wait_for_rendered_text(
child, captured, needle="Parent used the subagent result.", timeout=10
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert streaming_mock_server.requests[1].get("stream") is not True
assert_message_content_present(
streaming_mock_server.requests[1],
role="user",
expected=f"Find the marker {SUBAGENT_MARKER}.",
)
assert_tool_result_contains(
streaming_mock_server.requests[2],
call_id=SUBAGENT_TOOL_CALL_ID,
expected=f"Subagent found {SUBAGENT_MARKER}.",
)

219
tests/e2e/agent_loop_characterization/test_tool_execution.py Normal file
View file

@ -0,0 +1,219 @@
from __future__ import annotations
import os
from pathlib import Path
import pexpect
import pytest
from tests.e2e.agent_loop_characterization.support import (
answer_approval,
assert_assistant_tool_call_present,
assert_tool_result_contains,
assistant_text_chunks,
multi_tool_call_chunks,
set_tool_denylist,
single_tool_call_chunks,
wait_for_request_count_while_draining_child_output,
)
from tests.e2e.common import (
SpawnedVibeProcessFixture,
send_ctrl_c_until_quit_confirmation,
wait_for_main_screen,
wait_for_rendered_text,
wait_for_request_count,
)
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
DENIED_BASH_CALL_ID = "call_bash_denied"
DENIED_BASH_FILE = "denied-side-effect.txt"
FAILING_BASH_CALL_ID = "call_bash_fails"
FAILING_BASH_STDERR = "__E2E_BASH_FAILURE__"
MULTI_TOOL_FIRST_CALL_ID = "call_todo_multi_1"
MULTI_TOOL_SECOND_CALL_ID = "call_todo_multi_2"
def _denied_bash_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return single_tool_call_chunks(
call_id=DENIED_BASH_CALL_ID,
tool_name="bash",
arguments={"command": f"touch {DENIED_BASH_FILE}"},
)
return assistant_text_chunks("Denied without creating the file.", created=20)
def _failing_bash_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return single_tool_call_chunks(
call_id=FAILING_BASH_CALL_ID,
tool_name="bash",
arguments={"command": f"printf {FAILING_BASH_STDERR} >&2; false"},
)
return assistant_text_chunks("Recovered after the shell failure.", created=30)
def _multi_tool_turn_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return multi_tool_call_chunks(
[
(MULTI_TOOL_FIRST_CALL_ID, "todo", {"action": "read"}),
(MULTI_TOOL_SECOND_CALL_ID, "todo", {"action": "read"}),
],
created=160,
)
return assistant_text_chunks("Both todo reads completed.", created=170)
@pytest.mark.timeout(25)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_denied_bash_factory, id="denied-bash-tool")],
indirect=True,
)
def test_denylisted_bash_tool_does_not_run_and_is_reported_to_the_model(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
set_tool_denylist(Path(os.environ["VIBE_HOME"]), "bash", ["touch"])
denied_path = e2e_workdir / DENIED_BASH_FILE
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Try a denied shell command")
child.send("\r")
wait_for_request_count(
lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
)
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="Denied without creating the file.", timeout=10
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert not denied_path.exists()
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=DENIED_BASH_CALL_ID,
expected="Command denied:",
)
@pytest.mark.timeout(25)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_failing_bash_factory, id="failing-bash-tool")],
indirect=True,
)
def test_failed_bash_tool_result_is_reported_to_the_model_and_turn_recovers(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Run a failing shell command")
child.send("\r")
wait_for_request_count(
lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
)
answer_approval(child, captured, tool_name="bash", key="y")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="Recovered after the shell failure.", timeout=10
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=FAILING_BASH_CALL_ID,
expected="Return code: 1",
)
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=FAILING_BASH_CALL_ID,
expected=FAILING_BASH_STDERR,
)
@pytest.mark.timeout(25)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_multi_tool_turn_factory, id="multi-tool-turn")],
indirect=True,
)
def test_multiple_tool_calls_in_one_assistant_turn_return_distinct_results(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Run two todo reads")
child.send("\r")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="Both todo reads completed.", timeout=10
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert_assistant_tool_call_present(
streaming_mock_server.requests[1],
call_id=MULTI_TOOL_FIRST_CALL_ID,
tool_name="todo",
)
assert_assistant_tool_call_present(
streaming_mock_server.requests[1],
call_id=MULTI_TOOL_SECOND_CALL_ID,
tool_name="todo",
)
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=MULTI_TOOL_FIRST_CALL_ID,
expected="Retrieved 0 todos",
)
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=MULTI_TOOL_SECOND_CALL_ID,
expected="Retrieved 0 todos",
)

214
tests/e2e/agent_loop_characterization/test_tool_permissions.py Normal file
View file

@ -0,0 +1,214 @@
from __future__ import annotations
import os
from pathlib import Path
import pexpect
import pytest
from tests.e2e.agent_loop_characterization.support import (
answer_approval,
assert_tool_result_contains,
assistant_text_chunks,
set_tool_denylist,
single_tool_call_chunks,
wait_for_request_count_while_draining_child_output,
)
from tests.e2e.common import (
SpawnedVibeProcessFixture,
send_ctrl_c_until_quit_confirmation,
wait_for_main_screen,
wait_for_rendered_text,
wait_for_request_count,
)
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
WRITE_APPROVED_CALL_ID = "call_write_approved"
WRITE_REJECTED_CALL_ID = "call_write_rejected"
APPROVED_FILE = "approved-write.txt"
REJECTED_FILE = "rejected-write.txt"
SESSION_PERMISSION_FIRST_CALL_ID = "call_bash_session_1"
SESSION_PERMISSION_SECOND_CALL_ID = "call_bash_session_2"
SESSION_PERMISSION_OUTPUT = "__E2E_SESSION_PERMISSION__"
def _write_file_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return single_tool_call_chunks(
call_id=WRITE_APPROVED_CALL_ID,
tool_name="write_file",
arguments={"path": APPROVED_FILE, "content": "approved content\n"},
created=120,
)
if request_index == 1:
return assistant_text_chunks("Approved file was written.", created=130)
if request_index == 2:
return single_tool_call_chunks(
call_id=WRITE_REJECTED_CALL_ID,
tool_name="write_file",
arguments={"path": REJECTED_FILE, "content": "rejected content\n"},
created=140,
)
return assistant_text_chunks("Rejected file was not written.", created=150)
def _session_permission_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
arguments = {"command": f'printf "{SESSION_PERMISSION_OUTPUT}\\n"'}
if request_index == 0:
return single_tool_call_chunks(
call_id=SESSION_PERMISSION_FIRST_CALL_ID,
tool_name="bash",
arguments=arguments,
created=180,
)
if request_index == 1:
return assistant_text_chunks("First command completed.", created=190)
if request_index == 2:
return single_tool_call_chunks(
call_id=SESSION_PERMISSION_SECOND_CALL_ID,
tool_name="bash",
arguments=arguments,
created=200,
)
return assistant_text_chunks(
"Second command reused session permission.", created=210
)
@pytest.mark.timeout(35)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_write_file_factory, id="write-file-approval-and-rejection")],
indirect=True,
)
def test_write_file_approval_creates_file_and_rejection_leaves_file_absent(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
set_tool_denylist(
Path(os.environ["VIBE_HOME"]),
"write_file",
[str((e2e_workdir / REJECTED_FILE).resolve())],
)
approved_path = e2e_workdir / APPROVED_FILE
rejected_path = e2e_workdir / REJECTED_FILE
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Create the approved file")
child.send("\r")
wait_for_request_count(
lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
)
answer_approval(child, captured, tool_name="write_file", key="y")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="Approved file was written.", timeout=10
)
child.send("Try the rejected file")
child.send("\r")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=4,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="Rejected file was not written.", timeout=10
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert approved_path.read_text(encoding="utf-8") == "approved content\n"
assert not rejected_path.exists()
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=WRITE_APPROVED_CALL_ID,
expected="approved content",
)
assert_tool_result_contains(
streaming_mock_server.requests[3],
call_id=WRITE_REJECTED_CALL_ID,
expected="permanently disabled",
)
@pytest.mark.timeout(40)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_session_permission_factory, id="session-permission-memory")],
indirect=True,
)
def test_allow_for_session_reuses_bash_permission_without_prompting_again(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Run the first shell command")
child.send("\r")
wait_for_request_count(
lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
)
answer_approval(child, captured, tool_name="bash", key="2")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="First command completed.", timeout=10
)
child.send("Run the same shell command again")
child.send("\r")
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=4,
timeout=10,
)
wait_for_rendered_text(
child,
captured,
needle="Second command reused session permission.",
timeout=10,
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert_tool_result_contains(
streaming_mock_server.requests[1],
call_id=SESSION_PERMISSION_FIRST_CALL_ID,
expected=SESSION_PERMISSION_OUTPUT,
)
assert_tool_result_contains(
streaming_mock_server.requests[3],
call_id=SESSION_PERMISSION_SECOND_CALL_ID,
expected=SESSION_PERMISSION_OUTPUT,
)

100
tests/e2e/agent_loop_characterization/test_user_interaction.py Normal file
View file

@ -0,0 +1,100 @@
from __future__ import annotations
import io
from pathlib import Path
import time
import pexpect
import pytest
from tests.e2e.agent_loop_characterization.support import (
APPROVAL_INPUT_GRACE_PERIOD_S,
assert_tool_result_contains,
assistant_text_chunks,
single_tool_call_chunks,
wait_for_request_count_while_draining_child_output,
)
from tests.e2e.common import (
SpawnedVibeProcessFixture,
send_ctrl_c_until_quit_confirmation,
wait_for_main_screen,
wait_for_rendered_text,
wait_for_request_count,
)
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
QUESTION_CALL_ID = "call_question_mode"
QUESTION_TEXT = "Which mode should Vibe use?"
def _answer_first_question_option(child: pexpect.spawn, captured: io.StringIO) -> None:
wait_for_rendered_text(child, captured, needle=QUESTION_TEXT, timeout=10)
wait_for_rendered_text(child, captured, needle="Fast", timeout=10)
time.sleep(APPROVAL_INPUT_GRACE_PERIOD_S)
child.send("\r")
def _ask_user_question_factory(
request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
if request_index == 0:
return single_tool_call_chunks(
call_id=QUESTION_CALL_ID,
tool_name="ask_user_question",
arguments={
"questions": [
{
"question": QUESTION_TEXT,
"header": "Mode",
"options": [
{"label": "Fast", "description": "Move quickly"},
{"label": "Careful", "description": "Add checks"},
],
"hide_other": True,
}
]
},
created=70,
)
return assistant_text_chunks("The selected mode was Fast.", created=80)
@pytest.mark.timeout(25)
@pytest.mark.parametrize(
"streaming_mock_server",
[pytest.param(_ask_user_question_factory, id="ask-user-question")],
indirect=True,
)
def test_ask_user_question_waits_for_answer_and_reports_it_to_the_model(
streaming_mock_server: StreamingMockServer,
setup_e2e_env: None,
e2e_workdir: Path,
spawned_vibe_process: SpawnedVibeProcessFixture,
) -> None:
with spawned_vibe_process(e2e_workdir) as (child, captured):
wait_for_main_screen(child, timeout=15)
child.send("Ask me for a mode")
child.send("\r")
wait_for_request_count(
lambda: len(streaming_mock_server.requests), expected_count=1, timeout=10
)
_answer_first_question_option(child, captured)
wait_for_request_count_while_draining_child_output(
child,
captured,
lambda: len(streaming_mock_server.requests),
expected_count=2,
timeout=10,
)
wait_for_rendered_text(
child, captured, needle="The selected mode was Fast.", timeout=10
)
send_ctrl_c_until_quit_confirmation(child, captured, timeout=5)
child.expect(pexpect.EOF, timeout=10)
assert_tool_result_contains(
streaming_mock_server.requests[1], call_id=QUESTION_CALL_ID, expected="Fast"
)

123
tests/e2e/mock_server.py
View file

@ -8,6 +8,8 @@ import threading
import time
from typing import TypedDict, cast
from tests.constants import CHAT_COMPLETIONS_PATH
class StreamOptionsPayload(TypedDict, total=False):
include_usage: bool
@ -85,6 +87,104 @@ class StreamingMockServer:
),
]
@staticmethod
def _merge_tool_call_delta(
tool_calls_by_index: dict[int, dict[str, object]],
delta_tool_call: object,
*,
fallback_index: int | None,
) -> int | None:
if not isinstance(delta_tool_call, dict):
return fallback_index
index = delta_tool_call.get("index")
if not isinstance(index, int):
index = fallback_index
if index is None:
index = len(tool_calls_by_index)
tool_call = tool_calls_by_index.setdefault(index, {"index": index})
for key in ("id", "type"):
if value := delta_tool_call.get(key):
tool_call[key] = value
function_delta = delta_tool_call.get("function")
if not isinstance(function_delta, dict):
return index
function = tool_call.setdefault("function", {})
if not isinstance(function, dict):
function = {}
tool_call["function"] = function
if name := function_delta.get("name"):
function["name"] = name
if arguments := function_delta.get("arguments"):
function["arguments"] = f"{function.get('arguments', '')}{arguments}"
return index
@staticmethod
def _completion_response_from_chunks(
chunks: list[StreamChunk],
) -> dict[str, object]:
content_parts: list[str] = []
tool_calls_by_index: dict[int, dict[str, object]] = {}
active_tool_call_index: int | None = None
finish_reason: object = "stop"
usage: object = {"prompt_tokens": 3, "completion_tokens": 4}
created = 123
for chunk in chunks:
chunk_created = chunk.get("created")
if isinstance(chunk_created, int):
created = chunk_created
if chunk_usage := chunk.get("usage"):
usage = chunk_usage
choices = chunk.get("choices")
if not isinstance(choices, list) or not choices:
continue
choice = choices[0]
if not isinstance(choice, dict):
continue
if choice.get("finish_reason") is not None:
finish_reason = choice.get("finish_reason")
delta = choice.get("delta")
if not isinstance(delta, dict):
continue
if content := delta.get("content"):
content_parts.append(str(content))
if delta_tool_calls := delta.get("tool_calls"):
if isinstance(delta_tool_calls, list):
for delta_tool_call in delta_tool_calls:
active_tool_call_index = (
StreamingMockServer._merge_tool_call_delta(
tool_calls_by_index,
delta_tool_call,
fallback_index=active_tool_call_index,
)
)
message: dict[str, object] = {
"role": "assistant",
"content": "".join(content_parts),
}
tool_calls = [
tool_calls_by_index[index] for index in sorted(tool_calls_by_index)
]
if tool_calls:
message["tool_calls"] = tool_calls
return {
"id": "mock-id",
"object": "chat.completion",
"created": created,
"model": "mock-model",
"choices": [
{"index": 0, "message": message, "finish_reason": finish_reason}
],
"usage": usage,
}
def __init__(
self,
*,
@ -110,7 +210,7 @@ class StreamingMockServer:
return
def do_POST(self) -> None:
if self.path != "/v1/chat/completions":
if self.path != CHAT_COMPLETIONS_PATH:
self.send_response(404)
self.end_headers()
return
@ -125,17 +225,28 @@ class StreamingMockServer:
parent.requests.append(payload)
request_index = len(parent.requests) - 1
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.end_headers()
chunks = (
parent._chunk_factory(request_index, payload)
if parent._chunk_factory is not None
else parent._stream_chunks()
)
if not payload.get("stream"):
response = parent._completion_response_from_chunks(chunks)
response_body = json.dumps(response, ensure_ascii=False).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response_body)))
self.end_headers()
self.wfile.write(response_body)
self.wfile.flush()
return
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.end_headers()
for chunk in chunks:
data = json.dumps(chunk, ensure_ascii=False)
self.wfile.write(f"data: {data}\n\n".encode())

73
tests/e2e/test_mock_server.py Normal file
View file

@ -0,0 +1,73 @@
from __future__ import annotations
import httpx
from tests.e2e.mock_server import ChatCompletionsRequestPayload, StreamingMockServer
def _split_tool_call_factory(
_request_index: int, _payload: ChatCompletionsRequestPayload
) -> list[dict[str, object]]:
return [
StreamingMockServer.build_chunk(
created=10,
delta={
"role": "assistant",
"tool_calls": [
{
"index": 0,
"id": "call_split",
"type": "function",
"function": {"name": "bash", "arguments": '{"command":"echo'},
}
],
},
finish_reason=None,
),
StreamingMockServer.build_chunk(
created=11,
delta={"tool_calls": [{"function": {"arguments": ' ok"}'}}]},
finish_reason=None,
),
StreamingMockServer.build_chunk(
created=12,
delta={},
finish_reason="tool_calls",
usage={"prompt_tokens": 7, "completion_tokens": 8},
),
]
def test_non_streaming_completion_merges_split_tool_call_deltas() -> None:
server = StreamingMockServer(chunk_factory=_split_tool_call_factory)
server.start()
try:
response = httpx.post(
f"{server.api_base}/chat/completions",
json={"model": "mock-model", "messages": [], "stream": False},
timeout=5,
)
finally:
server.stop()
response.raise_for_status()
response_json = response.json()
choices = response_json["choices"]
assert isinstance(choices, list)
choice = choices[0]
assert isinstance(choice, dict)
message = choice["message"]
assert isinstance(message, dict)
tool_calls = message["tool_calls"]
assert isinstance(tool_calls, list)
assert tool_calls == [
{
"index": 0,
"id": "call_split",
"type": "function",
"function": {"name": "bash", "arguments": '{"command":"echo ok"}'},
}
]
assert choice["finish_reason"] == "tool_calls"
assert response_json["usage"] == {"prompt_tokens": 7, "completion_tokens": 8}
assert server.requests == [{"model": "mock-model", "messages": [], "stream": False}]