uvos/vibe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v2.5.0 (#495)

Browse source 
Co-authored-by: Quentin Torroba <quentin.torroba@mistral.ai>
Co-authored-by: Clement Sirieix <clem.sirieix@gmail.com>
Co-authored-by: Kim-Adeline Miguel <kimadeline.miguel@mistral.ai>
Co-authored-by: Simon Van de Kerckhove <simon.vandekerckhove@mistral.ai>
Co-authored-by: Vincent Guilloux <vincent.guilloux@mistral.ai>
Co-authored-by: Michel Thomazo <michel.thomazo@mistral.ai>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Mathias Gesbert 2026-03-16 17:51:47 +01:00 • committed by GitHub
parent 9421fbc08e
commit 5103019b01
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
104 changed files with 7277 additions and 691 deletions
Download patch file Download diff file Expand all files Collapse all files

377
tests/test_agent_tool_call.py
View file

@ -16,13 +16,13 @@ from vibe.core.config import VibeConfig
from vibe.core.tools.base import BaseToolConfig, ToolPermission
from vibe.core.tools.builtins.todo import TodoItem
from vibe.core.types import (
ApprovalCallback,
ApprovalResponse,
AssistantEvent,
BaseEvent,
FunctionCall,
LLMMessage,
Role,
SyncApprovalCallback,
ToolCall,
ToolCallEvent,
ToolResultEvent,
@ -58,7 +58,7 @@ def make_agent_loop(
auto_approve: bool = True,
todo_permission: ToolPermission = ToolPermission.ALWAYS,
backend: FakeBackend,
approval_callback: SyncApprovalCallback | None = None,
approval_callback: ApprovalCallback | None = None,
) -> AgentLoop:
agent_name = (
BuiltinAgentName.AUTO_APPROVE if auto_approve else BuiltinAgentName.DEFAULT
@ -165,7 +165,7 @@ async def test_tool_call_requires_approval_if_not_auto_approved(
@pytest.mark.asyncio
async def test_tool_call_approved_by_callback(telemetry_events: list[dict]) -> None:
def approval_callback(
async def approval_callback(
_tool_name: str, _args: BaseModel, _tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
return (ApprovalResponse.YES, None)
@ -209,7 +209,7 @@ async def test_tool_call_rejected_when_auto_approve_disabled_and_rejected_by_cal
) -> None:
custom_feedback = "User declined tool execution"
def approval_callback(
async def approval_callback(
_tool_name: str, _args: BaseModel, _tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
return (ApprovalResponse.NO, custom_feedback)
@ -237,6 +237,7 @@ async def test_tool_call_rejected_when_auto_approve_disabled_and_rejected_by_cal
assert events[3].error is None
assert events[3].result is None
assert events[3].skip_reason == custom_feedback
assert events[3].cancelled is False
assert agent_loop.stats.tool_calls_rejected == 1
assert agent_loop.stats.tool_calls_agreed == 0
assert agent_loop.stats.tool_calls_succeeded == 0
@ -297,7 +298,7 @@ async def test_approval_always_sets_tool_permission_for_subsequent_calls() -> No
callback_invocations = []
agent_ref: AgentLoop | None = None
def approval_callback(
async def approval_callback(
tool_name: str, _args: BaseModel, _tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
callback_invocations.append(tool_name)
@ -441,9 +442,9 @@ async def test_tool_call_with_exceeding_max_todos() -> None:
async def test_tool_call_can_be_interrupted() -> None:
"""Test that tool calls can be interrupted via asyncio.CancelledError.
Note: KeyboardInterrupt is no longer handled here as ctrl+C now quits the app directly.
When a tool raises CancelledError, the error is captured as a cancellation event
and the agent loop stops gracefully after the current tool batch completes.
"""
exception_class = asyncio.CancelledError
tool_call = ToolCall(
id="call_8", index=0, function=FunctionCall(name="stub_tool", arguments="{}")
)
@ -460,12 +461,11 @@ async def test_tool_call_can_be_interrupted() -> None:
agent_loop.tool_manager._available["stub_tool"] = FakeTool
stub_tool_instance = agent_loop.tool_manager.get("stub_tool")
assert isinstance(stub_tool_instance, FakeTool)
stub_tool_instance._exception_to_raise = exception_class()
stub_tool_instance._exception_to_raise = asyncio.CancelledError()
events: list[BaseEvent] = []
with pytest.raises(exception_class):
async for ev in agent_loop.act("Execute tool"):
events.append(ev)
async for ev in agent_loop.act("Execute tool"):
events.append(ev)
tool_result_event = next(
(e for e in events if isinstance(e, ToolResultEvent)), None
@ -473,6 +473,11 @@ async def test_tool_call_can_be_interrupted() -> None:
assert tool_result_event is not None
assert tool_result_event.error is not None
assert "execution interrupted by user" in tool_result_event.error.lower()
assert agent_loop.stats.tool_calls_failed == 1
# Agent loop should stop after cancellation — no second LLM turn
assistant_events = [e for e in events if isinstance(e, AssistantEvent)]
assert len(assistant_events) == 1
@pytest.mark.asyncio
@ -529,3 +534,353 @@ async def test_ensure_assistant_after_tool_appends_understood() -> None:
idx = next(i for i, m in enumerate(agent_loop.messages) if m.role == Role.tool)
assert agent_loop.messages[idx + 1].role == Role.assistant
assert agent_loop.messages[idx + 1].content == "Understood."
@pytest.mark.asyncio
async def test_parallel_tool_calls_produce_correct_events(
telemetry_events: list[dict],
) -> None:
"""Two tool calls in one LLM response should execute in parallel and produce correct events."""
tool_call_1 = make_todo_tool_call("call_p1", index=0)
tool_call_2 = make_todo_tool_call("call_p2", index=1)
backend = FakeBackend([
[
mock_llm_chunk(
content="Let me check two things.",
tool_calls=[tool_call_1, tool_call_2],
)
],
[mock_llm_chunk(content="Both done.")],
])
agent_loop = make_agent_loop(auto_approve=True, backend=backend)
events = await act_and_collect_events(agent_loop, "Check two things")
event_types = [type(e) for e in events]
# UserMessage, Assistant, ToolCall, ToolCall, then two ToolResults (order may vary), then Assistant
assert event_types[0] is UserMessageEvent
assert event_types[1] is AssistantEvent
# Both ToolCallEvents emitted upfront
assert event_types[2] is ToolCallEvent
assert event_types[3] is ToolCallEvent
tool_call_events = [e for e in events if isinstance(e, ToolCallEvent)]
assert {e.tool_call_id for e in tool_call_events} == {"call_p1", "call_p2"}
# Both ToolResultEvents present
tool_result_events = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_result_events) == 2
assert {e.tool_call_id for e in tool_result_events} == {"call_p1", "call_p2"}
for tool_result in tool_result_events:
assert tool_result.error is None
assert tool_result.skipped is False
assert tool_result.result is not None
# Final assistant message
assert event_types[-1] is AssistantEvent
assert isinstance(events[-1], AssistantEvent)
assert events[-1].content == "Both done."
# Verify conversation history has both tool responses
tool_msgs = [m for m in agent_loop.messages if m.role == Role.tool]
assert {m.tool_call_id for m in tool_msgs} == {"call_p1", "call_p2"}
assert agent_loop.stats.tool_calls_succeeded == 2
tool_finished = [
e for e in telemetry_events if e.get("event_name") == "vibe.tool_call_finished"
]
assert len(tool_finished) == 2
@pytest.mark.asyncio
async def test_parallel_tool_calls_with_approval_callback(
telemetry_events: list[dict],
) -> None:
"""Two parallel tool calls requiring approval should both succeed when approved."""
approval_calls: list[str] = []
async def approval_callback(
tool_name: str, _args: BaseModel, tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
approval_calls.append(tool_call_id)
return (ApprovalResponse.YES, None)
tool_call_1 = make_todo_tool_call("call_a1", index=0)
tool_call_2 = make_todo_tool_call("call_a2", index=1)
agent_loop = make_agent_loop(
auto_approve=False,
todo_permission=ToolPermission.ASK,
approval_callback=approval_callback,
backend=FakeBackend([
[
mock_llm_chunk(
content="Checking two.", tool_calls=[tool_call_1, tool_call_2]
)
],
[mock_llm_chunk(content="Both approved and done.")],
]),
)
events = await act_and_collect_events(agent_loop, "Check two things")
# Both should have been approved
assert set(approval_calls) == {"call_a1", "call_a2"}
tool_result_events = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_result_events) == 2
for tool_result in tool_result_events:
assert tool_result.error is None
assert tool_result.skipped is False
assert tool_result.result is not None
assert agent_loop.stats.tool_calls_agreed == 2
assert agent_loop.stats.tool_calls_succeeded == 2
@pytest.mark.asyncio
async def test_parallel_approvals_can_run_concurrently() -> None:
"""The core does not serialize approval callbacks — that is a CLI-layer concern.
Parallel tool calls may invoke the approval callback concurrently.
"""
concurrency = 0
max_concurrency = 0
async def approval_callback(
tool_name: str, _args: BaseModel, tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
nonlocal concurrency, max_concurrency
concurrency += 1
max_concurrency = max(max_concurrency, concurrency)
await asyncio.sleep(0.01)
concurrency -= 1
return (ApprovalResponse.YES, None)
tool_calls = [make_todo_tool_call(f"call_s{i}", index=i) for i in range(3)]
agent_loop = make_agent_loop(
auto_approve=False,
todo_permission=ToolPermission.ASK,
approval_callback=approval_callback,
backend=FakeBackend([
[mock_llm_chunk(content="Three tools.", tool_calls=tool_calls)],
[mock_llm_chunk(content="All done.")],
]),
)
await act_and_collect_events(agent_loop, "Go")
assert max_concurrency > 1
assert agent_loop.stats.tool_calls_agreed == 3
assert agent_loop.stats.tool_calls_succeeded == 3
@pytest.mark.asyncio
async def test_parallel_mixed_approval_and_rejection(
telemetry_events: list[dict],
) -> None:
"""One tool approved, one rejected — both should produce correct events."""
async def approval_callback(
tool_name: str, _args: BaseModel, tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
if tool_call_id == "call_yes":
return (ApprovalResponse.YES, None)
return (ApprovalResponse.NO, "Denied by user")
tc_yes = make_todo_tool_call("call_yes", index=0)
tc_no = make_todo_tool_call("call_no", index=1)
agent_loop = make_agent_loop(
auto_approve=False,
todo_permission=ToolPermission.ASK,
approval_callback=approval_callback,
backend=FakeBackend([
[mock_llm_chunk(content="Two tools.", tool_calls=[tc_yes, tc_no])],
[mock_llm_chunk(content="Mixed results.")],
]),
)
events = await act_and_collect_events(agent_loop, "Go")
tool_results = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_results) == 2
results_by_id = {e.tool_call_id: e for e in tool_results}
assert results_by_id["call_yes"].error is None
assert results_by_id["call_yes"].skipped is False
assert results_by_id["call_yes"].result is not None
assert results_by_id["call_no"].skipped is True
assert results_by_id["call_no"].skip_reason == "Denied by user"
assert results_by_id["call_no"].cancelled is False
assert agent_loop.stats.tool_calls_agreed == 1
assert agent_loop.stats.tool_calls_rejected == 1
assert agent_loop.stats.tool_calls_succeeded == 1
tool_finished = [
e for e in telemetry_events if e.get("event_name") == "vibe.tool_call_finished"
]
assert len(tool_finished) == 2
@pytest.mark.asyncio
async def test_parallel_three_tools_all_succeed(telemetry_events: list[dict]) -> None:
"""Three parallel tool calls should all complete successfully."""
tool_calls = [make_todo_tool_call(f"call_t{i}", index=i) for i in range(3)]
agent_loop = make_agent_loop(
auto_approve=True,
backend=FakeBackend([
[mock_llm_chunk(content="Three tools.", tool_calls=tool_calls)],
[mock_llm_chunk(content="All three done.")],
]),
)
events = await act_and_collect_events(agent_loop, "Go")
tool_call_events = [e for e in events if isinstance(e, ToolCallEvent)]
tool_result_events = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_call_events) == 3
assert len(tool_result_events) == 3
assert {e.tool_call_id for e in tool_call_events} == {
"call_t0",
"call_t1",
"call_t2",
}
assert {e.tool_call_id for e in tool_result_events} == {
"call_t0",
"call_t1",
"call_t2",
}
for tool_result in tool_result_events:
assert tool_result.error is None
assert tool_result.result is not None
assert agent_loop.stats.tool_calls_succeeded == 3
tool_msgs = [m for m in agent_loop.messages if m.role == Role.tool]
assert len(tool_msgs) == 3
tool_finished = [
e for e in telemetry_events if e.get("event_name") == "vibe.tool_call_finished"
]
assert len(tool_finished) == 3
@pytest.mark.asyncio
async def test_parallel_one_tool_error_does_not_block_others() -> None:
"""If one parallel tool fails, the other should still succeed."""
tc_good = make_todo_tool_call("call_good", index=0)
tc_bad = make_todo_tool_call(
"call_bad", index=1, arguments='{"action": "invalid_action"}'
)
agent_loop = make_agent_loop(
auto_approve=True,
backend=FakeBackend([
[mock_llm_chunk(content="Two tools.", tool_calls=[tc_good, tc_bad])],
[mock_llm_chunk(content="Done.")],
]),
)
events = await act_and_collect_events(agent_loop, "Go")
tool_results = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_results) == 2
results_by_id = {e.tool_call_id: e for e in tool_results}
assert results_by_id["call_good"].error is None
assert results_by_id["call_good"].result is not None
assert results_by_id["call_bad"].error is not None
assert results_by_id["call_bad"].result is None
assert agent_loop.stats.tool_calls_succeeded == 1
assert agent_loop.stats.tool_calls_failed == 1
@pytest.mark.asyncio
async def test_parallel_all_rejected_no_callback() -> None:
"""Parallel tools with no approval callback should all be skipped."""
tc1 = make_todo_tool_call("call_nc1", index=0)
tc2 = make_todo_tool_call("call_nc2", index=1)
agent_loop = make_agent_loop(
auto_approve=False,
todo_permission=ToolPermission.ASK,
backend=FakeBackend([
[mock_llm_chunk(content="Two tools.", tool_calls=[tc1, tc2])],
[mock_llm_chunk(content="Cannot proceed.")],
]),
)
events = await act_and_collect_events(agent_loop, "Go")
tool_results = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_results) == 2
for tool_result in tool_results:
assert tool_result.skipped is True
assert tool_result.result is None
assert agent_loop.stats.tool_calls_rejected == 2
assert agent_loop.stats.tool_calls_succeeded == 0
@pytest.mark.asyncio
async def test_parallel_all_permission_never() -> None:
"""Parallel tools with NEVER permission skip without calling the approval callback."""
approval_calls: list[str] = []
async def approval_callback(
tool_name: str, _args: BaseModel, tool_call_id: str
) -> tuple[ApprovalResponse, str | None]:
approval_calls.append(tool_call_id)
return (ApprovalResponse.YES, None)
tc1 = make_todo_tool_call("call_nv1", index=0)
tc2 = make_todo_tool_call("call_nv2", index=1)
agent_loop = make_agent_loop(
auto_approve=False,
todo_permission=ToolPermission.NEVER,
approval_callback=approval_callback,
backend=FakeBackend([
[mock_llm_chunk(content="Two tools.", tool_calls=[tc1, tc2])],
[mock_llm_chunk(content="Both disabled.")],
]),
)
events = await act_and_collect_events(agent_loop, "Go")
assert len(approval_calls) == 0
tool_results = [e for e in events if isinstance(e, ToolResultEvent)]
assert len(tool_results) == 2
for tool_result in tool_results:
assert tool_result.skipped is True
assert "permanently disabled" in (tool_result.skip_reason or "").lower()
assert agent_loop.stats.tool_calls_rejected == 2
@pytest.mark.asyncio
async def test_parallel_tool_call_events_emitted_before_results() -> None:
"""All ToolCallEvents must appear before any ToolResultEvent in the event stream."""
tool_calls = [make_todo_tool_call(f"call_o{i}", index=i) for i in range(3)]
agent_loop = make_agent_loop(
auto_approve=True,
backend=FakeBackend([
[mock_llm_chunk(content="Three tools.", tool_calls=tool_calls)],
[mock_llm_chunk(content="Done.")],
]),
)
events = await act_and_collect_events(agent_loop, "Go")
last_call_idx = max(i for i, e in enumerate(events) if isinstance(e, ToolCallEvent))
first_result_idx = min(
i for i, e in enumerate(events) if isinstance(e, ToolResultEvent)
)
assert last_call_idx < first_result_idx
@pytest.mark.asyncio
async def test_parallel_conversation_history_has_all_tool_messages() -> None:
"""All parallel tool results must appear in the conversation messages."""
tool_calls = [make_todo_tool_call(f"call_h{i}", index=i) for i in range(4)]
agent_loop = make_agent_loop(
auto_approve=True,
backend=FakeBackend([
[mock_llm_chunk(content="Four tools.", tool_calls=tool_calls)],
[mock_llm_chunk(content="All four done.")],
]),
)
await act_and_collect_events(agent_loop, "Go")
tool_msgs = [m for m in agent_loop.messages if m.role == Role.tool]
assert {m.tool_call_id for m in tool_msgs} == {
"call_h0",
"call_h1",
"call_h2",
"call_h3",
}
assert agent_loop.stats.tool_calls_succeeded == 4