vibe/tests/core/test_telemetry_send.py
Clément Drouin c2cb612ac1
v2.16.0 (#798)
Co-authored-by: Hdandria <henri.dandria@mistral.ai>
Co-authored-by: Liam Lyons <65613603+lyons-liam@users.noreply.github.com>
Co-authored-by: Mathias Gesbert <mathias.gesbert@mistral.ai>
Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai>
Co-authored-by: Quentin <quentin.torroba@mistral.ai>
Co-authored-by: allansimon-mistral <allan.simon@ext.mistral.ai>
Co-authored-by: angelapopopo <angele.lenglemetz@mistral.ai>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
2026-06-15 17:36:21 +02:00

1045 lines
39 KiB
Python

from __future__ import annotations
import asyncio
from pathlib import Path
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from pydantic import BaseModel
import pytest
from tests.conftest import build_test_vibe_config
from tests.stubs.fake_tool import FakeTool, FakeToolArgs
from vibe.core.agent_loop import ToolDecision, ToolExecutionResponse
from vibe.core.llm.format import ResolvedToolCall
from vibe.core.telemetry.build_metadata import (
build_base_metadata,
build_request_metadata,
)
from vibe.core.telemetry.send import TelemetryClient, _extract_file_extension
from vibe.core.telemetry.types import (
AttachmentKind,
EntrypointMetadata,
TelemetryRequestMetadata,
TerminalEmulator,
)
from vibe.core.tools.base import BaseTool, ToolPermission
from vibe.core.types import Backend
from vibe.core.utils import get_user_agent
_original_send_telemetry_event = TelemetryClient.send_telemetry_event
from vibe.core.tools.builtins.edit import Edit, EditArgs
from vibe.core.tools.builtins.read import Read, ReadArgs
from vibe.core.tools.builtins.write_file import WriteFile, WriteFileArgs
def _make_resolved_tool_call(
tool_name: str, args_dict: dict[str, Any]
) -> ResolvedToolCall:
validated: BaseModel
cls: type[BaseTool]
match tool_name:
case "write_file":
validated = WriteFileArgs(
path=args_dict.get("path", "foo.txt"), content="x"
)
cls = WriteFile
case "edit":
validated = EditArgs(
file_path=args_dict.get("file_path", "foo.txt"),
old_string="a",
new_string="b",
)
cls = Edit
case "read":
validated = ReadArgs(file_path=args_dict.get("file_path", "foo.txt"))
cls = Read
case _:
validated = FakeToolArgs()
cls = FakeTool
return ResolvedToolCall(
tool_name=tool_name, tool_class=cls, validated_args=validated, call_id="call_1"
)
def _run_telemetry_tasks() -> None:
loop = asyncio.new_event_loop()
try:
loop.run_until_complete(asyncio.sleep(0))
finally:
loop.close()
class TestExtractFileExtension:
@pytest.mark.parametrize(
("path", "expected"),
[
("/tmp/foo.py", ".py"),
("foo.TSX", ".tsx"),
("/tmp/Makefile", None),
("/tmp/.bashrc", None),
("archive.tar.gz", ".gz"),
("", None),
],
)
def test_string_inputs(self, path: str, expected: str | None) -> None:
assert _extract_file_extension(path) == expected
@pytest.mark.parametrize(
("path", "expected"),
[
(Path("/tmp/foo.py"), ".py"),
(Path("foo.TSX"), ".tsx"),
(Path("/tmp/Makefile"), None),
],
)
def test_path_inputs(self, path: Path, expected: str | None) -> None:
assert _extract_file_extension(path) == expected
@pytest.mark.parametrize("path", [None, 42, ["foo.py"], {"path": "foo.py"}])
def test_non_path_like_inputs_return_none(self, path: object) -> None:
assert _extract_file_extension(path) is None
class TestTelemetryClient:
def test_send_telemetry_event_swallows_config_getter_value_error(self) -> None:
def _raise_config_error() -> Any:
raise ValueError("config not ready")
client = TelemetryClient(config_getter=_raise_config_error)
client.send_telemetry_event("vibe.test", {})
def test_send_telemetry_event_does_nothing_when_api_key_is_none(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.delenv(env_key, raising=False)
client = TelemetryClient(config_getter=lambda: config)
assert client._get_mistral_api_key() is None
client._client = MagicMock()
client._client.post = AsyncMock()
client.send_telemetry_event("vibe.test", {})
_run_telemetry_tasks()
client._client.post.assert_not_called()
def test_send_telemetry_event_does_nothing_when_disabled(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=False)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
client = TelemetryClient(config_getter=lambda: config)
client._client = MagicMock()
client._client.post = AsyncMock()
client.send_telemetry_event("vibe.test", {})
_run_telemetry_tasks()
client._client.post.assert_not_called()
@pytest.mark.asyncio
async def test_send_telemetry_event_posts_when_enabled(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
client = TelemetryClient(config_getter=lambda: config)
mock_post = AsyncMock(return_value=MagicMock(status_code=204))
client._client = MagicMock()
client._client.post = mock_post
client._client.aclose = AsyncMock()
client.send_telemetry_event("vibe.test_event", {"key": "value"})
await client.aclose()
mock_post.assert_called_once_with(
"https://api.mistral.ai/v1/datalake/events",
json={"event": "vibe.test_event", "properties": {"key": "value"}},
headers={
"Content-Type": "application/json",
"Authorization": "Bearer sk-test",
"User-Agent": get_user_agent(Backend.MISTRAL),
},
)
def test_send_tool_call_finished_payload_shape(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("todo", {})
decision = ToolDecision(
verdict=ToolExecutionResponse.EXECUTE, approval_type=ToolPermission.ALWAYS
)
client.send_tool_call_finished(
tool_call=tool_call,
status="success",
decision=decision,
agent_profile_name="default",
model="mistral-large",
)
assert len(telemetry_events) == 1
event_name = telemetry_events[0]["event_name"]
assert event_name == "vibe.tool_call_finished"
properties = telemetry_events[0]["properties"]
assert properties["tool_name"] == "todo"
assert properties["status"] == "success"
assert properties["decision"] == "execute"
assert properties["approval_type"] == "always"
assert properties["agent_profile_name"] == "default"
assert properties["model"] == "mistral-large"
assert properties["nb_files_created"] == 0
assert properties["nb_files_modified"] == 0
assert properties["file_extension"] is None
assert properties["message_id"] is None
def test_send_tool_call_finished_with_message_id(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("todo", {})
client.send_tool_call_finished(
tool_call=tool_call,
status="success",
decision=None,
agent_profile_name="default",
model="mistral-large",
message_id="msg-123",
)
assert telemetry_events[0]["properties"]["message_id"] == "msg-123"
def test_send_tool_call_finished_nb_files_created_write_file(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("write_file", {"path": "/tmp/foo.PY"})
client.send_tool_call_finished(
tool_call=tool_call,
status="success",
decision=None,
agent_profile_name="default",
model="mistral-large",
result={},
)
assert telemetry_events[0]["properties"]["nb_files_created"] == 1
assert telemetry_events[0]["properties"]["nb_files_modified"] == 0
assert telemetry_events[0]["properties"]["file_extension"] == ".py"
def test_send_tool_call_finished_file_extension_edit(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call(
"edit", {"file_path": "/tmp/component.tsx"}
)
client.send_tool_call_finished(
tool_call=tool_call,
status="success",
decision=None,
agent_profile_name="default",
model="mistral-large",
result={},
)
assert telemetry_events[0]["properties"]["nb_files_modified"] == 1
assert telemetry_events[0]["properties"]["nb_files_created"] == 0
assert telemetry_events[0]["properties"]["file_extension"] == ".tsx"
def test_send_tool_call_finished_file_extension_read(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("read", {"file_path": "/tmp/lib.rs"})
client.send_tool_call_finished(
tool_call=tool_call,
status="success",
decision=None,
agent_profile_name="default",
model="mistral-large",
result={},
)
assert telemetry_events[0]["properties"]["nb_files_created"] == 0
assert telemetry_events[0]["properties"]["nb_files_modified"] == 0
assert telemetry_events[0]["properties"]["file_extension"] == ".rs"
def test_send_tool_call_finished_file_extension_none_when_no_suffix(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("write_file", {"path": "/tmp/Makefile"})
client.send_tool_call_finished(
tool_call=tool_call,
status="success",
decision=None,
agent_profile_name="default",
model="mistral-large",
result={},
)
assert telemetry_events[0]["properties"]["file_extension"] is None
def test_send_tool_call_finished_file_extension_none_on_failure(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("write_file", {"path": "/tmp/foo.py"})
client.send_tool_call_finished(
tool_call=tool_call,
status="failure",
decision=None,
agent_profile_name="default",
model="mistral-large",
result={},
)
assert telemetry_events[0]["properties"]["nb_files_created"] == 0
assert telemetry_events[0]["properties"]["file_extension"] is None
def test_send_tool_call_finished_decision_none(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
tool_call = _make_resolved_tool_call("todo", {})
client.send_tool_call_finished(
tool_call=tool_call,
status="skipped",
decision=None,
agent_profile_name="default",
model="mistral-large",
)
assert telemetry_events[0]["properties"]["decision"] is None
assert telemetry_events[0]["properties"]["approval_type"] is None
def test_send_user_copied_text_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_user_copied_text("hello world")
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.user_copied_text"
assert telemetry_events[0]["properties"]["text_length"] == 11
def test_send_user_cancelled_action_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_user_cancelled_action("interrupt_agent")
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.user_cancelled_action"
assert telemetry_events[0]["properties"]["action"] == "interrupt_agent"
def test_send_at_mention_inserted_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_at_mention_inserted(
nb_mentions=2,
context_types={"file": 1, "folder": 1},
file_extensions={".py": 1},
message_id="msg-123",
)
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.at_mention_inserted"
assert telemetry_events[0]["properties"] == {
"nb_mentions": 2,
"context_types": {"file": 1, "folder": 1},
"file_extensions": {".py": 1},
"message_id": "msg-123",
}
def test_send_at_mention_inserted_null_file_extensions(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_at_mention_inserted(
nb_mentions=1,
context_types={"folder": 1},
file_extensions=None,
message_id=None,
)
assert len(telemetry_events) == 1
assert telemetry_events[0]["properties"]["file_extensions"] is None
assert telemetry_events[0]["properties"]["message_id"] is None
def test_send_at_mention_inserted_multiple_files_counts_extensions(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_at_mention_inserted(
nb_mentions=4,
context_types={"file": 3, "folder": 1},
file_extensions={".py": 2, ".ts": 1},
message_id="msg-multi",
)
props = telemetry_events[0]["properties"]
assert props["nb_mentions"] == 4
assert props["context_types"] == {"file": 3, "folder": 1}
assert props["file_extensions"] == {".py": 2, ".ts": 1}
def test_send_at_mention_inserted_duplicate_references_reflected_in_counts(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_at_mention_inserted(
nb_mentions=4,
context_types={"file": 2, "folder": 2},
file_extensions={".py": 2},
message_id="msg-dupes",
)
props = telemetry_events[0]["properties"]
assert props["nb_mentions"] == 4
assert props["context_types"] == {"file": 2, "folder": 2}
assert props["file_extensions"] == {".py": 2}
def test_send_auto_compact_triggered_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_auto_compact_triggered(
nb_context_tokens_before=123, auto_compact_threshold=100, status="success"
)
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.auto_compact_triggered"
assert telemetry_events[0]["properties"] == {
"nb_context_tokens_before": 123,
"auto_compact_threshold": 100,
"status": "success",
}
def test_send_slash_command_used_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_slash_command_used("help", "builtin")
client.send_slash_command_used("my_skill", "skill")
assert len(telemetry_events) == 2
assert telemetry_events[0]["event_name"] == "vibe.slash_command_used"
assert telemetry_events[0]["properties"]["command"] == "help"
assert telemetry_events[0]["properties"]["command_type"] == "builtin"
assert telemetry_events[1]["properties"]["command"] == "my_skill"
assert telemetry_events[1]["properties"]["command_type"] == "skill"
def test_send_teleport_completed_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_teleport_completed(push_required=True, nb_session_messages=4)
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.teleport_completed"
assert telemetry_events[0]["properties"] == {
"push_required": True,
"nb_session_messages": 4,
}
def test_send_teleport_failed_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_teleport_failed(
stage="push",
error_class="ServiceTeleportError",
push_required=True,
nb_session_messages=4,
)
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.teleport_failed"
assert telemetry_events[0]["properties"] == {
"stage": "push",
"error_class": "ServiceTeleportError",
"push_required": True,
"nb_session_messages": 4,
}
def test_send_remote_resume_requested_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_remote_resume_requested(session_id="remote-123")
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.remote_resume_requested"
assert telemetry_events[0]["properties"] == {"session_id": "remote-123"}
def test_send_teleport_failed_payload_includes_error_details(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_teleport_failed(
stage="workflow_start",
error_class="ServiceTeleportError",
push_required=False,
nb_session_messages=4,
error_details={"failure_kind": "http_error", "http_status_code": 502},
)
assert telemetry_events[0]["event_name"] == "vibe.teleport_failed"
assert telemetry_events[0]["properties"] == {
"stage": "workflow_start",
"error_class": "ServiceTeleportError",
"push_required": False,
"nb_session_messages": 4,
"failure_kind": "http_error",
"http_status_code": 502,
}
def test_send_new_session_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_new_session(
has_agents_md=True,
nb_skills=2,
nb_mcp_servers=1,
nb_models=3,
entrypoint="cli",
client_name="vscode",
client_version="1.96.0",
terminal_emulator=TerminalEmulator.VSCODE,
)
assert len(telemetry_events) == 1
event_name = telemetry_events[0]["event_name"]
assert event_name == "vibe.new_session"
properties = telemetry_events[0]["properties"]
assert properties["has_agents_md"] is True
assert properties["nb_skills"] == 2
assert properties["nb_mcp_servers"] == 1
assert properties["nb_models"] == 3
assert properties["entrypoint"] == "cli"
assert properties["client_name"] == "vscode"
assert properties["client_version"] == "1.96.0"
assert properties["terminal_emulator"] == "vscode"
assert "version" in properties
@pytest.mark.asyncio
async def test_send_session_closed_payload(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
client = TelemetryClient(
config_getter=lambda: config,
session_id_getter=lambda: "current-session",
parent_session_id_getter=lambda: "current-parent-session",
entrypoint_metadata_getter=lambda: EntrypointMetadata(
agent_entrypoint="cli",
agent_version="1.0.0",
client_name="vibe_cli",
client_version="1.0.0",
),
)
mock_post = AsyncMock(return_value=MagicMock(status_code=204))
client._client = MagicMock()
client._client.post = mock_post
client._client.aclose = AsyncMock()
client.send_session_closed()
await client.aclose()
mock_post.assert_called_once_with(
"https://api.mistral.ai/v1/datalake/events",
json={
"event": "vibe.session_closed",
"properties": {
"agent_entrypoint": "cli",
"agent_version": "1.0.0",
"client_name": "vibe_cli",
"client_version": "1.0.0",
"session_id": "current-session",
"parent_session_id": "current-parent-session",
},
},
headers={
"Content-Type": "application/json",
"Authorization": "Bearer sk-test",
"User-Agent": get_user_agent(Backend.MISTRAL),
},
)
def test_build_base_metadata_includes_entrypoint_and_session(self) -> None:
metadata = build_base_metadata(
entrypoint_metadata=EntrypointMetadata(
agent_entrypoint="cli",
agent_version="1.0.0",
client_name="vibe_cli",
client_version="1.0.0",
),
session_id="session-123",
parent_session_id="parent-session-456",
)
assert metadata == {
"agent_entrypoint": "cli",
"agent_version": "1.0.0",
"client_name": "vibe_cli",
"client_version": "1.0.0",
"session_id": "session-123",
"parent_session_id": "parent-session-456",
}
def test_build_request_metadata_includes_all_telemetry_metadata(self) -> None:
metadata = build_request_metadata(
entrypoint_metadata=EntrypointMetadata(
agent_entrypoint="cli",
agent_version="1.0.0",
client_name="vibe_cli",
client_version="1.0.0",
),
session_id="session-123",
parent_session_id="parent-session-456",
call_type="secondary_call",
message_id="message-456",
)
assert metadata == TelemetryRequestMetadata(
agent_entrypoint="cli",
agent_version="1.0.0",
client_name="vibe_cli",
client_version="1.0.0",
session_id="session-123",
parent_session_id="parent-session-456",
call_source="vibe_code",
call_type="secondary_call",
message_id="message-456",
)
@pytest.mark.asyncio
async def test_parent_session_id_added_when_getter_provided(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_provider_for_model(
config.get_active_model()
).api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
client = TelemetryClient(
config_getter=lambda: config,
session_id_getter=lambda: "session-123",
parent_session_id_getter=lambda: "parent-session-456",
)
mock_post = AsyncMock(return_value=MagicMock(status_code=204))
client._client = MagicMock()
client._client.post = mock_post
client._client.aclose = AsyncMock()
client.send_telemetry_event("vibe.test_event", {"key": "value"})
await client.aclose()
mock_post.assert_called_once_with(
"https://api.mistral.ai/v1/datalake/events",
json={
"event": "vibe.test_event",
"properties": {
"session_id": "session-123",
"parent_session_id": "parent-session-456",
"key": "value",
},
},
headers={
"Content-Type": "application/json",
"Authorization": "Bearer sk-test",
"User-Agent": get_user_agent(Backend.MISTRAL),
},
)
@pytest.mark.asyncio
async def test_session_id_added_when_getter_provided(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
session_id = "test-session-uuid"
client = TelemetryClient(
config_getter=lambda: config, session_id_getter=lambda: session_id
)
mock_post = AsyncMock(return_value=MagicMock(status_code=204))
client._client = MagicMock()
client._client.post = mock_post
client._client.aclose = AsyncMock()
client.send_telemetry_event("vibe.test_event", {"key": "value"})
await client.aclose()
mock_post.assert_called_once_with(
"https://api.mistral.ai/v1/datalake/events",
json={
"event": "vibe.test_event",
"properties": {"session_id": session_id, "key": "value"},
},
headers={
"Content-Type": "application/json",
"Authorization": "Bearer sk-test",
"User-Agent": get_user_agent(Backend.MISTRAL),
},
)
@pytest.mark.asyncio
async def test_session_id_absent_when_no_getter(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
client = TelemetryClient(config_getter=lambda: config)
mock_post = AsyncMock(return_value=MagicMock(status_code=204))
client._client = MagicMock()
client._client.post = mock_post
client._client.aclose = AsyncMock()
client.send_telemetry_event("vibe.test_event", {"key": "value"})
await client.aclose()
mock_post.assert_called_once_with(
"https://api.mistral.ai/v1/datalake/events",
json={"event": "vibe.test_event", "properties": {"key": "value"}},
headers={
"Content-Type": "application/json",
"Authorization": "Bearer sk-test",
"User-Agent": get_user_agent(Backend.MISTRAL),
},
)
@pytest.mark.asyncio
async def test_session_id_getter_reflects_latest_value(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(
TelemetryClient, "send_telemetry_event", _original_send_telemetry_event
)
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.setenv(env_key, "sk-test")
current_id = "first-session-id"
client = TelemetryClient(
config_getter=lambda: config, session_id_getter=lambda: current_id
)
mock_post = AsyncMock(return_value=MagicMock(status_code=204))
client._client = MagicMock()
client._client.post = mock_post
client._client.aclose = AsyncMock()
client.send_telemetry_event("vibe.test_event", {})
current_id = "second-session-id"
client.send_telemetry_event("vibe.test_event", {})
await client.aclose()
calls = mock_post.call_args_list
assert calls[0].kwargs["json"]["properties"]["session_id"] == "first-session-id"
assert (
calls[1].kwargs["json"]["properties"]["session_id"] == "second-session-id"
)
def test_send_auto_compact_triggered_overrides_session_metadata(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(
config_getter=lambda: config,
session_id_getter=lambda: "new-session-id",
parent_session_id_getter=lambda: "new-parent-session-id",
)
client.send_auto_compact_triggered(
nb_context_tokens_before=123,
auto_compact_threshold=100,
status="success",
session_id="original-session-id",
parent_session_id=None,
)
assert len(telemetry_events) == 1
properties = telemetry_events[0]["properties"]
assert properties["session_id"] == "original-session-id"
assert properties["parent_session_id"] is None
def test_send_ready_payload(self, telemetry_events: list[dict[str, Any]]) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_ready(init_duration_ms=1240)
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.ready"
assert telemetry_events[0]["properties"]["init_duration_ms"] == 1240
def test_send_request_sent_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_request_sent(
model="codestral",
nb_context_chars=1234,
nb_context_messages=5,
nb_prompt_chars=42,
call_type="main_call",
)
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.request_sent"
properties = telemetry_events[0]["properties"]
assert properties["model"] == "codestral"
assert properties["nb_context_chars"] == 1234
assert properties["nb_context_messages"] == 5
assert properties["nb_prompt_chars"] == 42
assert properties["call_source"] == "vibe_code"
assert properties["call_type"] == "main_call"
assert properties["message_id"] is None
assert properties["attachment_counts"] == {}
def test_send_request_sent_payload_with_attachments(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_request_sent(
model="codestral",
nb_context_chars=1234,
nb_context_messages=5,
nb_prompt_chars=42,
call_type="main_call",
attachment_counts={AttachmentKind.IMAGE: 2},
)
assert len(telemetry_events) == 1
properties = telemetry_events[0]["properties"]
assert properties["attachment_counts"] == {"image": 2}
def test_send_request_sent_payload_drops_zero_counts(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_request_sent(
model="codestral",
nb_context_chars=1234,
nb_context_messages=5,
nb_prompt_chars=42,
call_type="main_call",
attachment_counts={AttachmentKind.IMAGE: 0},
)
assert telemetry_events[0]["properties"]["attachment_counts"] == {}
def test_send_user_rating_feedback_payload(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_user_rating_feedback(rating=2, model="mistral-large")
assert len(telemetry_events) == 1
assert telemetry_events[0]["event_name"] == "vibe.user_rating_feedback"
properties = telemetry_events[0]["properties"]
assert properties["rating"] == 2
assert properties["model"] == "mistral-large"
assert "version" in properties
def test_send_user_rating_feedback_includes_correlation_id(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.last_correlation_id = "corr-abc-123"
client.send_user_rating_feedback(rating=1, model="mistral-large")
assert len(telemetry_events) == 1
assert telemetry_events[0]["correlation_id"] == "corr-abc-123"
def test_send_user_rating_feedback_omits_correlation_id_when_none(
self, telemetry_events: list[dict[str, Any]]
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
client = TelemetryClient(config_getter=lambda: config)
client.send_user_rating_feedback(rating=1, model="mistral-large")
assert len(telemetry_events) == 1
assert "correlation_id" not in telemetry_events[0]
def test_telemetry_url_custom_provider_config(self) -> None:
from vibe.core.config import ProviderConfig
from vibe.core.types import Backend
custom_api_base = "https://api.custom.host/v2"
config = build_test_vibe_config(
enable_telemetry=True,
providers=[
ProviderConfig(
name="mistral",
api_base=custom_api_base,
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
],
)
client = TelemetryClient(config_getter=lambda: config)
assert (
client._get_telemetry_url(custom_api_base)
== "https://api.custom.host/v1/datalake/events"
)
def test_telemetry_url_preserves_port_in_api_base(self) -> None:
from vibe.core.config import ProviderConfig
from vibe.core.types import Backend
custom_api_base = "http://api.custom.host:8080/v1/"
config = build_test_vibe_config(
enable_telemetry=True,
providers=[
ProviderConfig(
name="mistral",
api_base=custom_api_base,
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
],
)
client = TelemetryClient(config_getter=lambda: config)
assert (
client._get_telemetry_url(custom_api_base)
== "http://api.custom.host:8080/v1/datalake/events"
)
def test_telemetry_url_falls_back_to_default_when_api_base_malformed(self) -> None:
from vibe.core.config import ProviderConfig
from vibe.core.types import Backend
config = build_test_vibe_config(
enable_telemetry=True,
providers=[
ProviderConfig(
name="mistral",
api_base="not-a-valid-url",
api_key_env_var="MISTRAL_API_KEY",
backend=Backend.MISTRAL,
)
],
)
client = TelemetryClient(config_getter=lambda: config)
assert (
client._get_telemetry_url("not-a-valid-url")
== "https://api.mistral.ai/v1/datalake/events"
)
def test_is_active_false_when_mistral_provider_exists_but_no_api_key(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
config = build_test_vibe_config(enable_telemetry=True)
env_key = config.get_active_provider().api_key_env_var
monkeypatch.delenv(env_key, raising=False)
client = TelemetryClient(config_getter=lambda: config)
assert client.is_active() is False
def test_is_active_false_when_no_mistral_provider(self) -> None:
from vibe.core.config import ProviderConfig
config = build_test_vibe_config(
enable_telemetry=True,
providers=[
ProviderConfig(
name="llamacpp",
api_base="http://127.0.0.1:8080/v1",
api_key_env_var="",
)
],
)
client = TelemetryClient(config_getter=lambda: config)
assert client.is_active() is False
def test_is_active_false_when_config_getter_raises_value_error(self) -> None:
def _raise_config_error() -> Any:
raise ValueError("config not ready")
client = TelemetryClient(config_getter=_raise_config_error)
assert client.is_active() is False