from __future__ import annotations import asyncio from pathlib import Path from typing import cast from unittest.mock import AsyncMock, patch import pytest from tests.conftest import ( build_test_agent_loop, build_test_vibe_config, make_test_models, ) from tests.mock.utils import mock_llm_chunk from tests.stubs.fake_backend import FakeBackend from vibe.core.agent_loop import CompactionFailedError from vibe.core.config import ModelConfig from vibe.core.types import ( AssistantEvent, CompactEndEvent, CompactStartEvent, FunctionCall, LLMMessage, Role, ToolCall, UserMessageEvent, ) def _get_auto_compact_properties( telemetry_events: list[dict[str, object]], ) -> dict[str, object]: auto_compact = [ event for event in telemetry_events if event.get("event_name") == "vibe.auto_compact_triggered" ] assert len(auto_compact) == 1 return cast(dict[str, object], auto_compact[0]["properties"]) @pytest.mark.asyncio async def test_auto_compact_emits_correct_events(telemetry_events: list[dict]) -> None: backend = FakeBackend([ [mock_llm_chunk(content="")], [mock_llm_chunk(content="")], ]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.stats.context_tokens = 2 old_session_id = agent.session_id events = [ev async for ev in agent.act("Hello")] assert len(events) == 4 assert isinstance(events[0], UserMessageEvent) assert isinstance(events[1], CompactStartEvent) assert isinstance(events[2], CompactEndEvent) assert isinstance(events[3], AssistantEvent) start: CompactStartEvent = events[1] end: CompactEndEvent = events[2] final: AssistantEvent = events[3] assert start.current_context_tokens == 2 assert start.threshold == 1 assert isinstance(end, CompactEndEvent) assert final.content == "" properties = _get_auto_compact_properties(telemetry_events) assert properties["nb_context_tokens_before"] == 2 assert properties["auto_compact_threshold"] == 1 assert properties["status"] == "success" assert properties["session_id"] == old_session_id assert properties["parent_session_id"] is None @pytest.mark.asyncio @pytest.mark.parametrize( ("side_effect", "expected_exception", "match", "expected_status"), [ pytest.param( RuntimeError("boom"), RuntimeError, "boom", "failure", id="failure" ), pytest.param( asyncio.CancelledError(), asyncio.CancelledError, None, "cancelled", id="cancelled", ), ], ) async def test_auto_compact_emits_terminal_telemetry( side_effect: BaseException, expected_exception: type[BaseException], match: str | None, expected_status: str, telemetry_events: list[dict], ) -> None: backend = FakeBackend([[mock_llm_chunk(content="")]]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.stats.context_tokens = 2 old_session_id = agent.session_id events = [] with patch.object(agent, "compact", AsyncMock(side_effect=side_effect)): if match is None: with pytest.raises(expected_exception): async for event in agent.act("Hello"): events.append(event) else: with pytest.raises(expected_exception, match=match): async for event in agent.act("Hello"): events.append(event) assert len(events) == 2 assert isinstance(events[0], UserMessageEvent) assert isinstance(events[1], CompactStartEvent) properties = _get_auto_compact_properties(telemetry_events) assert properties["nb_context_tokens_before"] == 2 assert properties["auto_compact_threshold"] == 1 assert properties["status"] == expected_status assert properties["session_id"] == old_session_id assert properties["parent_session_id"] is None @pytest.mark.asyncio async def test_auto_compact_observer_sees_user_msg_not_summary() -> None: """Observer sees the original user message and final response. Compact internals (summary request, LLM summary) are invisible to the observer because they happen inside silent() / reset(). """ observed: list[tuple[Role, str | None]] = [] def observer(msg: LLMMessage) -> None: observed.append((msg.role, msg.content)) backend = FakeBackend([ [mock_llm_chunk(content="")], [mock_llm_chunk(content="")], ]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1)) agent = build_test_agent_loop( config=cfg, message_observer=observer, backend=backend ) agent.stats.context_tokens = 2 [_ async for _ in agent.act("Hello")] roles = [r for r, _ in observed] assert roles == [Role.system, Role.user, Role.assistant] assert observed[1][1] == "Hello" assert observed[2][1] == "" @pytest.mark.asyncio async def test_auto_compact_observer_does_not_see_summary_request() -> None: """The compact summary request and LLM response must not leak to observer.""" observed: list[tuple[Role, str | None]] = [] def observer(msg: LLMMessage) -> None: observed.append((msg.role, msg.content)) backend = FakeBackend([ [mock_llm_chunk(content="")], [mock_llm_chunk(content="")], ]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1)) agent = build_test_agent_loop( config=cfg, message_observer=observer, backend=backend ) agent.stats.context_tokens = 2 [_ async for _ in agent.act("Hello")] contents = [c for _, c in observed] assert "" not in contents assert all("compact" not in (c or "").lower() for c in contents) @pytest.mark.asyncio async def test_compact_replaces_messages_with_context() -> None: backend = FakeBackend([ [mock_llm_chunk(content="")], [mock_llm_chunk(content="")], ]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.stats.context_tokens = 2 [_ async for _ in agent.act("Hello")] # After compact + final response: system, compaction context, final. assert agent.messages[0].role == Role.system assert agent.messages[-1].role == Role.assistant assert agent.messages[-1].content == "" class _ModelTrackingBackend(FakeBackend): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.requested_models: list[ModelConfig] = [] async def complete(self, *, model, **kwargs): self.requested_models.append(model) return await super().complete(model=model, **kwargs) @pytest.mark.asyncio async def test_compact_uses_compaction_model() -> None: """When compaction_model is set, compact() uses it instead of active_model.""" compaction = ModelConfig( name="compaction-model", provider="mistral", alias="compaction", auto_compact_threshold=1, ) backend = _ModelTrackingBackend([ [mock_llm_chunk(content="")], [mock_llm_chunk(content="")], ]) cfg = build_test_vibe_config( models=make_test_models(auto_compact_threshold=1), compaction_model=compaction ) agent = build_test_agent_loop(config=cfg, backend=backend) agent.stats.context_tokens = 2 [_ async for _ in agent.act("Hello")] assert backend.requested_models[0].name == "compaction-model" assert backend.requested_models[1].name != "compaction-model" @pytest.mark.asyncio async def test_compact_uses_active_model_when_no_compaction_model() -> None: """Without compaction_model, compact() falls back to the active model.""" backend = _ModelTrackingBackend([ [mock_llm_chunk(content="")], [mock_llm_chunk(content="")], ]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=1)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.stats.context_tokens = 2 [_ async for _ in agent.act("Hello")] active = cfg.get_active_model() assert backend.requested_models[0].name == active.name assert backend.requested_models[1].name == active.name @pytest.mark.asyncio async def test_compact_appends_extra_instructions_to_prompt() -> None: backend = FakeBackend([[mock_llm_chunk(content="")]]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="Hello")) agent.stats.context_tokens = 100 await agent.compact(extra_instructions="focus on auth") compaction_prompt = backend.requests_messages[0][-1].content assert compaction_prompt is not None assert "## Additional Instructions" in compaction_prompt assert "focus on auth" in compaction_prompt @pytest.mark.asyncio async def test_compact_uses_configured_compaction_prompt( mock_prompts_dirs: tuple[Path, Path], ) -> None: project_prompts, _ = mock_prompts_dirs (project_prompts / "theorem_compact.md").write_text("Summarize theorem progress") backend = FakeBackend([[mock_llm_chunk(content="")]]) cfg = build_test_vibe_config( models=make_test_models(auto_compact_threshold=999), compaction_prompt_id="theorem_compact", ) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="Hello")) agent.stats.context_tokens = 100 await agent.compact() compaction_prompt = backend.requests_messages[0][-1].content assert compaction_prompt == "Summarize theorem progress" @pytest.mark.asyncio async def test_compact_without_extra_instructions_has_no_additional_section() -> None: backend = FakeBackend([[mock_llm_chunk(content="")]]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="Hello")) agent.stats.context_tokens = 100 await agent.compact() compaction_prompt = backend.requests_messages[0][-1].content assert compaction_prompt is not None assert "## Additional Instructions" not in compaction_prompt @pytest.mark.asyncio async def test_compact_raises_on_tool_call_when_flag_enabled() -> None: """With the flag on, a compaction that returns a tool call raises.""" backend = FakeBackend([ [ mock_llm_chunk( content="", tool_calls=[ ToolCall( id="t1", index=0, function=FunctionCall(name="bash", arguments="{}"), ) ], ) ] ]) cfg = build_test_vibe_config( models=make_test_models(auto_compact_threshold=999), raise_on_compaction_failure=True, ) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="Hello")) agent.stats.context_tokens = 100 with pytest.raises(CompactionFailedError) as exc_info: await agent.compact() assert exc_info.value.reason == "tool_call" @pytest.mark.asyncio async def test_compact_raises_on_empty_summary_when_flag_enabled() -> None: """With the flag on, a compaction with empty content raises.""" backend = FakeBackend([[mock_llm_chunk(content=" ")]]) cfg = build_test_vibe_config( models=make_test_models(auto_compact_threshold=999), raise_on_compaction_failure=True, ) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="Hello")) agent.stats.context_tokens = 100 with pytest.raises(CompactionFailedError) as exc_info: await agent.compact() assert exc_info.value.reason == "empty_summary" @pytest.mark.asyncio async def test_compact_falls_back_when_flag_disabled() -> None: """With the flag off (default), empty content uses the legacy fallback.""" backend = FakeBackend([[mock_llm_chunk(content="")]]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="Hello")) agent.stats.context_tokens = 100 summary = await agent.compact() assert summary == "(no summary available)" @pytest.mark.asyncio async def test_compact_message_shape_preserves_prior_user_messages() -> None: from vibe.core.compaction import parse_previous_user_messages from vibe.core.prompts import UtilityPrompt summary_prefix = UtilityPrompt.COMPACT_SUMMARY_PREFIX.read() backend = FakeBackend([[mock_llm_chunk(content="fresh summary body")]]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999)) agent = build_test_agent_loop(config=cfg, backend=backend) system_message_before = agent.messages[0] agent.messages.append(LLMMessage(role=Role.user, content="first real ask")) agent.messages.append( LLMMessage(role=Role.user, content="middleware ping", injected=True) ) agent.messages.append(LLMMessage(role=Role.assistant, content="ack")) agent.messages.append( LLMMessage( role=Role.user, content=f"{summary_prefix}\nprior summary blob", injected=True, ) ) agent.messages.append(LLMMessage(role=Role.user, content="follow-up ask")) agent.stats.context_tokens = 100 await agent.compact() final = list(agent.messages) assert len(final) == 2 # [system, compaction_context] assert final[0] is system_message_before assert final[1].role == Role.user assert final[1].injected is True assert parse_previous_user_messages(final[1].content or "") == [ "first real ask", "follow-up ask", ] assert "Here are some of the most recent previous user messages" in ( final[1].content or "" ) assert "" in (final[1].content or "") assert "fresh summary body" in (final[1].content or "") # Injected and prior-summary user messages must be filtered out. assert all("middleware ping" not in (m.content or "") for m in final) assert sum("prior summary blob" in (m.content or "") for m in final) == 0 @pytest.mark.asyncio async def test_compact_preserves_user_messages_across_repeated_compactions() -> None: from vibe.core.compaction import parse_previous_user_messages backend = FakeBackend([ [mock_llm_chunk(content="summary one")], [mock_llm_chunk(content="summary two")], ]) cfg = build_test_vibe_config(models=make_test_models(auto_compact_threshold=999)) agent = build_test_agent_loop(config=cfg, backend=backend) agent.messages.append(LLMMessage(role=Role.user, content="first ask")) agent.stats.context_tokens = 100 await agent.compact() agent.messages.append(LLMMessage(role=Role.user, content="second ask")) agent.stats.context_tokens = 100 await agent.compact() final = list(agent.messages) assert len(final) == 2 assert parse_previous_user_messages(final[1].content or "") == [ "first ask", "second ask", ]