v2.10.0 (#697)
Co-authored-by: Clément Drouin <clement.drouin@mistral.ai> Co-authored-by: Corentin André <corentin.andre@mistral.ai> Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com> Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com> Co-authored-by: Maxime Dolores <maxime.dolores@ext.mistral.ai> Co-authored-by: Nelson PROIA <144663685+Nelson-PROIA@users.noreply.github.com> Co-authored-by: Peter Evers <pevers90@gmail.com> Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai> Co-authored-by: Quentin <quentin.torroba@mistral.ai> Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com> Co-authored-by: MichisGitIsKing <MichisGitIsKing@users.noreply.github.com> Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Mathias Gesbert
GitHub
parent
626f905186
commit
228f3c65a9
158 changed files with 7235 additions and 916 deletions
0
tests/core/experiments/__init__.py
Normal file
0
tests/core/experiments/__init__.py
Normal file
116
tests/core/experiments/test_client.py
Normal file
116
tests/core/experiments/test_client.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from vibe.core.experiments._constants import build_eval_url
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.models import ExperimentAttributes
|
||||
|
||||
_TEST_API_HOST = "https://growthbook.test"
|
||||
_TEST_CLIENT_KEY = "sdk-test"
|
||||
_TEST_EVAL_URL = build_eval_url(_TEST_API_HOST, _TEST_CLIENT_KEY)
|
||||
assert _TEST_EVAL_URL is not None
|
||||
|
||||
|
||||
def _attrs() -> ExperimentAttributes:
|
||||
return ExperimentAttributes(
|
||||
userId="hashed",
|
||||
entrypoint="cli",
|
||||
agent_version="1.2.3",
|
||||
client_name="vibe",
|
||||
client_version="1.2.3",
|
||||
os="darwin",
|
||||
)
|
||||
|
||||
|
||||
def _make_client() -> RemoteEvalClient:
|
||||
return RemoteEvalClient.from_settings(_TEST_API_HOST, _TEST_CLIENT_KEY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_happy_path() -> None:
|
||||
payload = {
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
}
|
||||
}
|
||||
route = respx.post(_TEST_EVAL_URL).mock(
|
||||
return_value=httpx.Response(200, json=payload)
|
||||
)
|
||||
client = _make_client()
|
||||
response = await client.evaluate(_attrs())
|
||||
await client.aclose()
|
||||
|
||||
assert route.called
|
||||
request_body = route.calls.last.request.read()
|
||||
assert b'"userId":"hashed"' in request_body
|
||||
assert b'"forcedVariations":{}' in request_body
|
||||
assert b'"forcedFeatures":[]' in request_body
|
||||
assert response is not None
|
||||
assert (
|
||||
response.features[ExperimentName.SYSTEM_PROMPT.value].resolved_value()
|
||||
== "cli_v2"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _silence_vibe_logger(caplog: pytest.LogCaptureFixture) -> None:
|
||||
# Failure-path tests legitimately emit WARNING logs. Silence the vibe
|
||||
# logger so they don't leak into ~/.vibe/logs/vibe.log (the file handler
|
||||
# is bound to the real path at module import, before VIBE_HOME is
|
||||
# monkeypatched in the conftest fixture).
|
||||
caplog.set_level(logging.CRITICAL, logger="vibe")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_returns_none_on_5xx(_silence_vibe_logger: None) -> None:
|
||||
respx.post(_TEST_EVAL_URL).mock(return_value=httpx.Response(500, text="oops"))
|
||||
client = _make_client()
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_returns_none_on_network_error(
|
||||
_silence_vibe_logger: None,
|
||||
) -> None:
|
||||
respx.post(_TEST_EVAL_URL).mock(side_effect=httpx.ConnectError("nope"))
|
||||
client = _make_client()
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_returns_none_on_invalid_json(
|
||||
_silence_vibe_logger: None,
|
||||
) -> None:
|
||||
respx.post(_TEST_EVAL_URL).mock(return_value=httpx.Response(200, text="not json"))
|
||||
client = _make_client()
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluate_skips_request_when_url_unset() -> None:
|
||||
client = RemoteEvalClient.from_settings(api_host="", client_key="sdk-x")
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclose_is_idempotent() -> None:
|
||||
client = _make_client()
|
||||
await client.aclose()
|
||||
await client.aclose()
|
||||
427
tests/core/experiments/test_manager.py
Normal file
427
tests/core/experiments/test_manager.py
Normal file
|
|
@ -0,0 +1,427 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vibe.core.experiments.active import DEFAULT_VARIANTS, ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager, hash_api_key
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
self.calls: list[ExperimentAttributes] = []
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
self.calls.append(attributes)
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _attrs() -> ExperimentAttributes:
|
||||
return ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
|
||||
|
||||
def _response(features: dict[str, Any]) -> EvalResponse:
|
||||
return EvalResponse.model_validate({"features": features})
|
||||
|
||||
|
||||
def test_hash_api_key_is_stable_and_anonymous() -> None:
|
||||
a = hash_api_key("sk-abc")
|
||||
b = hash_api_key("sk-abc")
|
||||
assert a == b
|
||||
assert "sk-" not in a
|
||||
assert len(a) == 32
|
||||
|
||||
|
||||
def test_hash_api_key_differs_per_key() -> None:
|
||||
assert hash_api_key("sk-abc") != hash_api_key("sk-def")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_returns_default_when_uninitialized() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_or_none_returns_none_when_unassigned() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
assert manager.get_variant_or_none(ExperimentName.SYSTEM_PROMPT) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_or_none_returns_override() -> None:
|
||||
manager = ExperimentManager(
|
||||
client=_StubClient(None), overrides={ExperimentName.SYSTEM_PROMPT.value: "lean"}
|
||||
)
|
||||
assert manager.get_variant_or_none(ExperimentName.SYSTEM_PROMPT) == "lean"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_or_none_returns_resolved_value() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "explore", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant_or_none(ExperimentName.SYSTEM_PROMPT) == "explore"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_returns_resolved_value() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli_v2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overrides_take_precedence_over_remote() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(
|
||||
client=_StubClient(response),
|
||||
overrides={ExperimentName.SYSTEM_PROMPT.value: "forced"},
|
||||
)
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "forced"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_falls_back_when_remote_returns_no_match() -> None:
|
||||
response = _response({"some_other_feature": {"defaultValue": True}})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_uses_resolved_variant_value() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assignments = manager.assignments()
|
||||
assert assignments == {ExperimentName.SYSTEM_PROMPT.value: "cli_v2"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_prefers_track_result_value() -> None:
|
||||
# When the GrowthBook payload carries an explicit per-arm value, it wins
|
||||
# over the rule's force value — it's the most precise label for the user.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"value": "cli_v3",
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.assignments() == {ExperimentName.SYSTEM_PROMPT.value: "cli_v3"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_keys_on_feature_id_when_experiment_key_differs() -> None:
|
||||
# GrowthBook feature ID and experiment key can diverge. Telemetry must
|
||||
# key on the feature ID (matches ExperimentName) so overrides, lookups
|
||||
# and downstream analysis stay consistent.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": "vibe-code-cli-system-prompt"},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
"featureId": ExperimentName.SYSTEM_PROMPT.value,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assignments = manager.assignments()
|
||||
assert assignments == {ExperimentName.SYSTEM_PROMPT.value: "cli_v2"}
|
||||
assert "vibe-code-cli-system-prompt" not in assignments
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_does_nothing_on_failed_eval() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.assignments() == {}
|
||||
for name in ExperimentName:
|
||||
assert manager.get_variant(name) == DEFAULT_VARIANTS[name]
|
||||
|
||||
|
||||
def test_experiment_attributes_default_custom_system_prompt_to_false() -> None:
|
||||
attrs = ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
assert attrs.custom_system_prompt is False
|
||||
assert attrs.model_dump(exclude_none=True)["custom_system_prompt"] is False
|
||||
|
||||
|
||||
def test_experiment_attributes_serializes_custom_system_prompt() -> None:
|
||||
attrs = ExperimentAttributes(
|
||||
userId="x",
|
||||
entrypoint="cli",
|
||||
agent_version="0",
|
||||
os="darwin",
|
||||
custom_system_prompt=True,
|
||||
)
|
||||
assert attrs.model_dump(exclude_none=True)["custom_system_prompt"] is True
|
||||
|
||||
|
||||
def test_export_state_returns_none_before_initialize() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
assert manager.export_state() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_export_state_returns_response_after_initialize() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.export_state() == response
|
||||
|
||||
|
||||
def test_hydrate_does_not_call_client() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
stub = _StubClient(response)
|
||||
manager = ExperimentManager(client=stub)
|
||||
manager.hydrate(response)
|
||||
assert stub.calls == []
|
||||
|
||||
|
||||
def test_hydrate_makes_get_variant_match_initialized_manager() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
hydrated = ExperimentManager(client=_StubClient(None))
|
||||
hydrated.hydrate(response)
|
||||
assert hydrated.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli_v2"
|
||||
assert hydrated.assignments() == {ExperimentName.SYSTEM_PROMPT.value: "cli_v2"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_twice_replaces_response() -> None:
|
||||
# Session reset re-runs initialize_experiments, which calls manager.initialize
|
||||
# again. The second response must replace the first so the new session sees
|
||||
# the up-to-date variant assignment.
|
||||
first = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "explore", "tracks": []}],
|
||||
}
|
||||
})
|
||||
second = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "lean", "tracks": []}],
|
||||
}
|
||||
})
|
||||
|
||||
class _ScriptedClient(RemoteEvalClient):
|
||||
def __init__(self, responses: list[EvalResponse]) -> None:
|
||||
self._responses = responses
|
||||
self.call_count = 0
|
||||
|
||||
async def evaluate(
|
||||
self, attributes: ExperimentAttributes
|
||||
) -> EvalResponse | None:
|
||||
response = self._responses[self.call_count]
|
||||
self.call_count += 1
|
||||
return response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
client = _ScriptedClient([first, second])
|
||||
manager = ExperimentManager(client=client)
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "explore"
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "lean"
|
||||
assert client.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_excludes_tracks_not_in_experiment() -> None:
|
||||
# Holdouts and forced overrides come back with inExperiment=False.
|
||||
# We must NOT report them as active experiment participants — that would
|
||||
# pollute downstream A/B analysis since the assignment map IS the
|
||||
# exposure record (no separate exposure event).
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "0",
|
||||
"variationId": 0,
|
||||
"inExperiment": False,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
|
||||
assert manager.assignments() == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_excludes_tracks_with_missing_in_experiment() -> None:
|
||||
# Defensive: if the proxy returns a track without inExperiment, treat it
|
||||
# as "not confirmed in the experiment" rather than silently counting it.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {"key": "1", "variationId": 1},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
|
||||
assert manager.assignments() == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_drops_unknown_features() -> None:
|
||||
# GrowthBook returns every feature defined in the org. Only the ones
|
||||
# listed in ExperimentName should survive into manager state and the
|
||||
# persisted snapshot.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {"defaultValue": "cli"},
|
||||
"unrelated_org_flag": {"defaultValue": True},
|
||||
"neko": {"defaultValue": False},
|
||||
"gbdemo-checkout-layout": {"defaultValue": "dev"},
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
|
||||
snapshot = manager.export_state()
|
||||
assert snapshot is not None
|
||||
assert set(snapshot.features.keys()) == {ExperimentName.SYSTEM_PROMPT.value}
|
||||
|
||||
|
||||
def test_hydrate_drops_unknown_features() -> None:
|
||||
# When resuming a session saved by an older vibe version, the snapshot
|
||||
# may contain experiments that have since been retired. Filter them on
|
||||
# hydrate so the in-memory state matches the current ExperimentName.
|
||||
legacy_response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {"defaultValue": "cli"},
|
||||
"retired_experiment": {"defaultValue": "control"},
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
manager.hydrate(legacy_response)
|
||||
|
||||
snapshot = manager.export_state()
|
||||
assert snapshot is not None
|
||||
assert set(snapshot.features.keys()) == {ExperimentName.SYSTEM_PROMPT.value}
|
||||
68
tests/core/experiments/test_models.py
Normal file
68
tests/core/experiments/test_models.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.models import EvalResponse, FeatureDefinition, FeatureRule
|
||||
|
||||
|
||||
class TestFeatureDefinition:
|
||||
def test_resolved_value_returns_force_when_present(self) -> None:
|
||||
feature = FeatureDefinition(
|
||||
defaultValue="cli", rules=[FeatureRule(force="cli_v2")]
|
||||
)
|
||||
assert feature.resolved_value() == "cli_v2"
|
||||
|
||||
def test_resolved_value_falls_back_to_default(self) -> None:
|
||||
feature = FeatureDefinition(defaultValue="cli")
|
||||
assert feature.resolved_value() == "cli"
|
||||
|
||||
def test_resolved_value_picks_first_rule_with_force(self) -> None:
|
||||
feature = FeatureDefinition(
|
||||
defaultValue="cli",
|
||||
rules=[FeatureRule(force=None), FeatureRule(force="cli_v2")],
|
||||
)
|
||||
assert feature.resolved_value() == "cli_v2"
|
||||
|
||||
|
||||
class TestEvalResponse:
|
||||
def test_parses_real_proxy_payload(self) -> None:
|
||||
# A reduced version of the live response captured during dev probing.
|
||||
sp_key = ExperimentName.SYSTEM_PROMPT.value
|
||||
raw = {
|
||||
"features": {
|
||||
"rbac_enabled": {"defaultValue": True},
|
||||
sp_key: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {
|
||||
"key": sp_key,
|
||||
"variations": [{}, "cli_v2"],
|
||||
},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"value": "cli_v2",
|
||||
"inExperiment": True,
|
||||
"hashAttribute": "id",
|
||||
"hashValue": "abc",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
"experiments": [],
|
||||
"dateUpdated": "2026-05-11T13:29:48.394Z",
|
||||
}
|
||||
response = EvalResponse.model_validate(raw)
|
||||
assert response.features["rbac_enabled"].defaultValue is True
|
||||
sp = response.features[sp_key]
|
||||
assert sp.resolved_value() == "cli_v2"
|
||||
track = sp.rules[0].tracks[0]
|
||||
assert track.experiment.key == sp_key
|
||||
assert track.result.variationId == 1
|
||||
assert track.result.inExperiment is True
|
||||
168
tests/core/experiments/test_resume.py
Normal file
168
tests/core/experiments/test_resume.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_vibe_config
|
||||
from vibe.core.agents import AgentManager
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
from vibe.core.session.session_loader import SessionLoader
|
||||
from vibe.core.skills.manager import SkillManager
|
||||
from vibe.core.system_prompt import get_universal_system_prompt
|
||||
from vibe.core.tools.manager import ToolManager
|
||||
from vibe.core.types import SessionMetadata
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
self.calls: list[ExperimentAttributes] = []
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
self.calls.append(attributes)
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _attrs() -> ExperimentAttributes:
|
||||
return ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
|
||||
|
||||
def _response_forcing(variant: str) -> EvalResponse:
|
||||
return EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": variant,
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {
|
||||
"key": ExperimentName.SYSTEM_PROMPT.value
|
||||
},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_round_trip_preserves_variant() -> None:
|
||||
response = _response_forcing("explore")
|
||||
fresh = ExperimentManager(client=_StubClient(response))
|
||||
await fresh.initialize(_attrs())
|
||||
|
||||
persisted_state = fresh.export_state()
|
||||
assert persisted_state is not None
|
||||
|
||||
serialized = persisted_state.model_dump_json()
|
||||
rehydrated_response = EvalResponse.model_validate_json(serialized)
|
||||
|
||||
stub = _StubClient(None)
|
||||
resumed = ExperimentManager(client=stub)
|
||||
resumed.hydrate(rehydrated_response)
|
||||
|
||||
assert resumed.get_variant(ExperimentName.SYSTEM_PROMPT) == "explore"
|
||||
assert resumed.assignments() == fresh.assignments()
|
||||
assert stub.calls == []
|
||||
|
||||
|
||||
def test_resume_old_session_without_experiments_field_falls_back_to_defaults(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
legacy_metadata = {
|
||||
"session_id": "legacy",
|
||||
"start_time": "2025-01-01T00:00:00+00:00",
|
||||
"end_time": None,
|
||||
"git_commit": None,
|
||||
"git_branch": None,
|
||||
"environment": {"working_directory": str(tmp_path)},
|
||||
"username": "tester",
|
||||
"loops": [],
|
||||
}
|
||||
meta_path = tmp_path / "meta.json"
|
||||
meta_path.write_text(json.dumps(legacy_metadata))
|
||||
|
||||
metadata = SessionLoader.load_metadata(tmp_path)
|
||||
|
||||
assert metadata.experiments is None
|
||||
|
||||
|
||||
def test_resume_overrides_still_win_over_hydrated_state() -> None:
|
||||
response = _response_forcing("explore")
|
||||
manager = ExperimentManager(
|
||||
client=_StubClient(None), overrides={ExperimentName.SYSTEM_PROMPT.value: "lean"}
|
||||
)
|
||||
manager.hydrate(response)
|
||||
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "lean"
|
||||
|
||||
|
||||
def test_session_metadata_round_trips_experiments_field() -> None:
|
||||
response = _response_forcing("explore")
|
||||
metadata = SessionMetadata(
|
||||
session_id="s",
|
||||
start_time="2025-01-01T00:00:00+00:00",
|
||||
end_time=None,
|
||||
git_commit=None,
|
||||
git_branch=None,
|
||||
environment={"working_directory": "/tmp"},
|
||||
username="tester",
|
||||
experiments=response,
|
||||
)
|
||||
|
||||
serialized = metadata.model_dump_json()
|
||||
restored = SessionMetadata.model_validate_json(serialized)
|
||||
|
||||
assert restored.experiments is not None
|
||||
assert restored.experiments == response
|
||||
|
||||
|
||||
def _build_managers(config):
|
||||
return (
|
||||
ToolManager(lambda: config),
|
||||
SkillManager(lambda: config),
|
||||
AgentManager(lambda: config),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_graduated_experiment_with_deleted_variant_file_falls_back() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = _response_forcing("removed_after_graduation_2025_07")
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
manager.hydrate(response)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
|
||||
default_prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=None
|
||||
)
|
||||
assert prompt == default_prompt
|
||||
208
tests/core/experiments/test_session_helpers.py
Normal file
208
tests/core/experiments/test_session_helpers.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
from vibe.core.experiments.session import (
|
||||
hydrate_experiments_from_session,
|
||||
initialize_experiments,
|
||||
)
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _make_config(
|
||||
*, enable_telemetry: bool = True, enable_experiments: bool = True
|
||||
) -> Any:
|
||||
config = MagicMock()
|
||||
config.enable_telemetry = enable_telemetry
|
||||
config.experiments.enable = enable_experiments
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_telemetry_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(enable_telemetry=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_experiments_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# `experiments.enable=False` must short-circuit even when telemetry is on
|
||||
# — it is the user's opt-out for being assigned to A/B tests without
|
||||
# disabling all telemetry.
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(enable_experiments=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_no_mistral_provider(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session.get_mistral_provider_and_api_key",
|
||||
lambda _config: None,
|
||||
)
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_remote_eval_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# Regression: even if telemetry is enabled and a Mistral key is set,
|
||||
# a failed remote eval (returns None) leaves manager state empty —
|
||||
# the helper must NOT report success, so the caller skips the
|
||||
# unnecessary system prompt refresh.
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session.get_mistral_provider_and_api_key",
|
||||
lambda _config: (MagicMock(), "fake-key"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session._build_attributes",
|
||||
lambda *_args, **_kwargs: ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
),
|
||||
)
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_true_and_persists_when_remote_eval_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session.get_mistral_provider_and_api_key",
|
||||
lambda _config: (MagicMock(), "fake-key"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session._build_attributes",
|
||||
lambda *_args, **_kwargs: ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
),
|
||||
)
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
persist.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hydrate_returns_false_when_telemetry_disabled() -> None:
|
||||
session_logger = MagicMock()
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
|
||||
})
|
||||
session_logger.session_metadata.experiments = response
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await hydrate_experiments_from_session(
|
||||
config=_make_config(enable_telemetry=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert manager.export_state() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hydrate_returns_false_when_experiments_disabled() -> None:
|
||||
# Regression: without this gate, a user who flipped experiments.enable
|
||||
# to False between sessions would still resume into a hydrated variant.
|
||||
session_logger = MagicMock()
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
|
||||
})
|
||||
session_logger.session_metadata.experiments = response
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await hydrate_experiments_from_session(
|
||||
config=_make_config(enable_experiments=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert manager.export_state() is None
|
||||
186
tests/core/experiments/test_system_prompt_variant.py
Normal file
186
tests/core/experiments/test_system_prompt_variant.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_vibe_config
|
||||
from vibe.core.agents import AgentManager
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
from vibe.core.prompts import load_system_prompt
|
||||
from vibe.core.skills.manager import SkillManager
|
||||
from vibe.core.system_prompt import get_universal_system_prompt
|
||||
from vibe.core.tools.manager import ToolManager
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _build_managers(config):
|
||||
return (
|
||||
ToolManager(lambda: config),
|
||||
SkillManager(lambda: config),
|
||||
AgentManager(lambda: config),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_prompt_uses_assigned_variant() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "tests", "tracks": []}],
|
||||
}
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt.startswith("You are Vibe, a super useful programming assistant.")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_prompt_falls_back_to_default_when_variant_unknown() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "nonexistent_prompt", "tracks": []}],
|
||||
}
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
# Falls back to the default `cli` system prompt.
|
||||
assert "You are Mistral Vibe" in prompt or "You are Vibe" in prompt
|
||||
|
||||
|
||||
def test_system_prompt_uses_default_when_no_manager() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager
|
||||
)
|
||||
assert "You are Mistral Vibe" in prompt or "You are Vibe" in prompt
|
||||
|
||||
|
||||
def test_system_prompt_honors_user_config_when_manager_uninitialized() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="lean",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt == load_system_prompt("lean")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_prompt_honors_user_config_when_no_remote_assignment() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="lean",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"some_other_feature": {"defaultValue": True}}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt == load_system_prompt("lean")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_overrides_assigned_experiment_variant() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="lean",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "explore", "tracks": []}],
|
||||
}
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt == load_system_prompt("lean")
|
||||
assert prompt != load_system_prompt("explore")
|
||||
31
tests/core/experiments/test_telemetry_integration.py
Normal file
31
tests/core/experiments/test_telemetry_integration.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from tests.conftest import build_test_vibe_config
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.telemetry.send import TelemetryClient
|
||||
|
||||
|
||||
def test_build_client_event_metadata_omits_experiments_when_empty() -> None:
|
||||
config = build_test_vibe_config(enable_telemetry=True)
|
||||
client = TelemetryClient(
|
||||
config_getter=lambda: config, experiments_getter=lambda: {}
|
||||
)
|
||||
metadata = client.build_client_event_metadata()
|
||||
assert "experiments" not in metadata
|
||||
|
||||
|
||||
def test_build_client_event_metadata_includes_experiments_when_present() -> None:
|
||||
config = build_test_vibe_config(enable_telemetry=True)
|
||||
sp_key = ExperimentName.SYSTEM_PROMPT.value
|
||||
client = TelemetryClient(
|
||||
config_getter=lambda: config, experiments_getter=lambda: {sp_key: "1"}
|
||||
)
|
||||
metadata = client.build_client_event_metadata()
|
||||
assert metadata["experiments"] == {sp_key: "1"}
|
||||
|
||||
|
||||
def test_build_client_event_metadata_works_without_getter() -> None:
|
||||
config = build_test_vibe_config(enable_telemetry=True)
|
||||
client = TelemetryClient(config_getter=lambda: config)
|
||||
metadata = client.build_client_event_metadata()
|
||||
assert "experiments" not in metadata
|
||||
Loading…
Add table
Add a link
Reference in a new issue