v2.10.0 (#697)
Co-authored-by: Clément Drouin <clement.drouin@mistral.ai> Co-authored-by: Corentin André <corentin.andre@mistral.ai> Co-authored-by: Guillaume LE GOFF <guillaume.lgf@gmail.com> Co-authored-by: Kim-Adeline Miguel <51720070+kimadeline@users.noreply.github.com> Co-authored-by: Maxime Dolores <maxime.dolores@ext.mistral.ai> Co-authored-by: Nelson PROIA <144663685+Nelson-PROIA@users.noreply.github.com> Co-authored-by: Peter Evers <pevers90@gmail.com> Co-authored-by: Pierre Rossinès <pierre.rossines@mistral.ai> Co-authored-by: Quentin <quentin.torroba@mistral.ai> Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com> Co-authored-by: MichisGitIsKing <MichisGitIsKing@users.noreply.github.com> Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Mathias Gesbert
GitHub
parent
626f905186
commit
228f3c65a9
158 changed files with 7235 additions and 916 deletions
0
tests/core/experiments/__init__.py
Normal file
0
tests/core/experiments/__init__.py
Normal file
116
tests/core/experiments/test_client.py
Normal file
116
tests/core/experiments/test_client.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from vibe.core.experiments._constants import build_eval_url
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.models import ExperimentAttributes
|
||||
|
||||
_TEST_API_HOST = "https://growthbook.test"
|
||||
_TEST_CLIENT_KEY = "sdk-test"
|
||||
_TEST_EVAL_URL = build_eval_url(_TEST_API_HOST, _TEST_CLIENT_KEY)
|
||||
assert _TEST_EVAL_URL is not None
|
||||
|
||||
|
||||
def _attrs() -> ExperimentAttributes:
|
||||
return ExperimentAttributes(
|
||||
userId="hashed",
|
||||
entrypoint="cli",
|
||||
agent_version="1.2.3",
|
||||
client_name="vibe",
|
||||
client_version="1.2.3",
|
||||
os="darwin",
|
||||
)
|
||||
|
||||
|
||||
def _make_client() -> RemoteEvalClient:
|
||||
return RemoteEvalClient.from_settings(_TEST_API_HOST, _TEST_CLIENT_KEY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_happy_path() -> None:
|
||||
payload = {
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
}
|
||||
}
|
||||
route = respx.post(_TEST_EVAL_URL).mock(
|
||||
return_value=httpx.Response(200, json=payload)
|
||||
)
|
||||
client = _make_client()
|
||||
response = await client.evaluate(_attrs())
|
||||
await client.aclose()
|
||||
|
||||
assert route.called
|
||||
request_body = route.calls.last.request.read()
|
||||
assert b'"userId":"hashed"' in request_body
|
||||
assert b'"forcedVariations":{}' in request_body
|
||||
assert b'"forcedFeatures":[]' in request_body
|
||||
assert response is not None
|
||||
assert (
|
||||
response.features[ExperimentName.SYSTEM_PROMPT.value].resolved_value()
|
||||
== "cli_v2"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _silence_vibe_logger(caplog: pytest.LogCaptureFixture) -> None:
|
||||
# Failure-path tests legitimately emit WARNING logs. Silence the vibe
|
||||
# logger so they don't leak into ~/.vibe/logs/vibe.log (the file handler
|
||||
# is bound to the real path at module import, before VIBE_HOME is
|
||||
# monkeypatched in the conftest fixture).
|
||||
caplog.set_level(logging.CRITICAL, logger="vibe")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_returns_none_on_5xx(_silence_vibe_logger: None) -> None:
|
||||
respx.post(_TEST_EVAL_URL).mock(return_value=httpx.Response(500, text="oops"))
|
||||
client = _make_client()
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_returns_none_on_network_error(
|
||||
_silence_vibe_logger: None,
|
||||
) -> None:
|
||||
respx.post(_TEST_EVAL_URL).mock(side_effect=httpx.ConnectError("nope"))
|
||||
client = _make_client()
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_evaluate_returns_none_on_invalid_json(
|
||||
_silence_vibe_logger: None,
|
||||
) -> None:
|
||||
respx.post(_TEST_EVAL_URL).mock(return_value=httpx.Response(200, text="not json"))
|
||||
client = _make_client()
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_evaluate_skips_request_when_url_unset() -> None:
|
||||
client = RemoteEvalClient.from_settings(api_host="", client_key="sdk-x")
|
||||
assert await client.evaluate(_attrs()) is None
|
||||
await client.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aclose_is_idempotent() -> None:
|
||||
client = _make_client()
|
||||
await client.aclose()
|
||||
await client.aclose()
|
||||
427
tests/core/experiments/test_manager.py
Normal file
427
tests/core/experiments/test_manager.py
Normal file
|
|
@ -0,0 +1,427 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vibe.core.experiments.active import DEFAULT_VARIANTS, ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager, hash_api_key
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
self.calls: list[ExperimentAttributes] = []
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
self.calls.append(attributes)
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _attrs() -> ExperimentAttributes:
|
||||
return ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
|
||||
|
||||
def _response(features: dict[str, Any]) -> EvalResponse:
|
||||
return EvalResponse.model_validate({"features": features})
|
||||
|
||||
|
||||
def test_hash_api_key_is_stable_and_anonymous() -> None:
|
||||
a = hash_api_key("sk-abc")
|
||||
b = hash_api_key("sk-abc")
|
||||
assert a == b
|
||||
assert "sk-" not in a
|
||||
assert len(a) == 32
|
||||
|
||||
|
||||
def test_hash_api_key_differs_per_key() -> None:
|
||||
assert hash_api_key("sk-abc") != hash_api_key("sk-def")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_returns_default_when_uninitialized() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_or_none_returns_none_when_unassigned() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
assert manager.get_variant_or_none(ExperimentName.SYSTEM_PROMPT) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_or_none_returns_override() -> None:
|
||||
manager = ExperimentManager(
|
||||
client=_StubClient(None), overrides={ExperimentName.SYSTEM_PROMPT.value: "lean"}
|
||||
)
|
||||
assert manager.get_variant_or_none(ExperimentName.SYSTEM_PROMPT) == "lean"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_or_none_returns_resolved_value() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "explore", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant_or_none(ExperimentName.SYSTEM_PROMPT) == "explore"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_returns_resolved_value() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli_v2"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_overrides_take_precedence_over_remote() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(
|
||||
client=_StubClient(response),
|
||||
overrides={ExperimentName.SYSTEM_PROMPT.value: "forced"},
|
||||
)
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "forced"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_variant_falls_back_when_remote_returns_no_match() -> None:
|
||||
response = _response({"some_other_feature": {"defaultValue": True}})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_uses_resolved_variant_value() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assignments = manager.assignments()
|
||||
assert assignments == {ExperimentName.SYSTEM_PROMPT.value: "cli_v2"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_prefers_track_result_value() -> None:
|
||||
# When the GrowthBook payload carries an explicit per-arm value, it wins
|
||||
# over the rule's force value — it's the most precise label for the user.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"value": "cli_v3",
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.assignments() == {ExperimentName.SYSTEM_PROMPT.value: "cli_v3"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_keys_on_feature_id_when_experiment_key_differs() -> None:
|
||||
# GrowthBook feature ID and experiment key can diverge. Telemetry must
|
||||
# key on the feature ID (matches ExperimentName) so overrides, lookups
|
||||
# and downstream analysis stay consistent.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": "vibe-code-cli-system-prompt"},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
"featureId": ExperimentName.SYSTEM_PROMPT.value,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assignments = manager.assignments()
|
||||
assert assignments == {ExperimentName.SYSTEM_PROMPT.value: "cli_v2"}
|
||||
assert "vibe-code-cli-system-prompt" not in assignments
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_does_nothing_on_failed_eval() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.assignments() == {}
|
||||
for name in ExperimentName:
|
||||
assert manager.get_variant(name) == DEFAULT_VARIANTS[name]
|
||||
|
||||
|
||||
def test_experiment_attributes_default_custom_system_prompt_to_false() -> None:
|
||||
attrs = ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
assert attrs.custom_system_prompt is False
|
||||
assert attrs.model_dump(exclude_none=True)["custom_system_prompt"] is False
|
||||
|
||||
|
||||
def test_experiment_attributes_serializes_custom_system_prompt() -> None:
|
||||
attrs = ExperimentAttributes(
|
||||
userId="x",
|
||||
entrypoint="cli",
|
||||
agent_version="0",
|
||||
os="darwin",
|
||||
custom_system_prompt=True,
|
||||
)
|
||||
assert attrs.model_dump(exclude_none=True)["custom_system_prompt"] is True
|
||||
|
||||
|
||||
def test_export_state_returns_none_before_initialize() -> None:
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
assert manager.export_state() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_export_state_returns_response_after_initialize() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.export_state() == response
|
||||
|
||||
|
||||
def test_hydrate_does_not_call_client() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "cli_v2", "tracks": []}],
|
||||
}
|
||||
})
|
||||
stub = _StubClient(response)
|
||||
manager = ExperimentManager(client=stub)
|
||||
manager.hydrate(response)
|
||||
assert stub.calls == []
|
||||
|
||||
|
||||
def test_hydrate_makes_get_variant_match_initialized_manager() -> None:
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
hydrated = ExperimentManager(client=_StubClient(None))
|
||||
hydrated.hydrate(response)
|
||||
assert hydrated.get_variant(ExperimentName.SYSTEM_PROMPT) == "cli_v2"
|
||||
assert hydrated.assignments() == {ExperimentName.SYSTEM_PROMPT.value: "cli_v2"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_twice_replaces_response() -> None:
|
||||
# Session reset re-runs initialize_experiments, which calls manager.initialize
|
||||
# again. The second response must replace the first so the new session sees
|
||||
# the up-to-date variant assignment.
|
||||
first = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "explore", "tracks": []}],
|
||||
}
|
||||
})
|
||||
second = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "lean", "tracks": []}],
|
||||
}
|
||||
})
|
||||
|
||||
class _ScriptedClient(RemoteEvalClient):
|
||||
def __init__(self, responses: list[EvalResponse]) -> None:
|
||||
self._responses = responses
|
||||
self.call_count = 0
|
||||
|
||||
async def evaluate(
|
||||
self, attributes: ExperimentAttributes
|
||||
) -> EvalResponse | None:
|
||||
response = self._responses[self.call_count]
|
||||
self.call_count += 1
|
||||
return response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
client = _ScriptedClient([first, second])
|
||||
manager = ExperimentManager(client=client)
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "explore"
|
||||
await manager.initialize(_attrs())
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "lean"
|
||||
assert client.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_excludes_tracks_not_in_experiment() -> None:
|
||||
# Holdouts and forced overrides come back with inExperiment=False.
|
||||
# We must NOT report them as active experiment participants — that would
|
||||
# pollute downstream A/B analysis since the assignment map IS the
|
||||
# exposure record (no separate exposure event).
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {
|
||||
"key": "0",
|
||||
"variationId": 0,
|
||||
"inExperiment": False,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
|
||||
assert manager.assignments() == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assignments_excludes_tracks_with_missing_in_experiment() -> None:
|
||||
# Defensive: if the proxy returns a track without inExperiment, treat it
|
||||
# as "not confirmed in the experiment" rather than silently counting it.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {"key": ExperimentName.SYSTEM_PROMPT.value},
|
||||
"result": {"key": "1", "variationId": 1},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
|
||||
assert manager.assignments() == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_drops_unknown_features() -> None:
|
||||
# GrowthBook returns every feature defined in the org. Only the ones
|
||||
# listed in ExperimentName should survive into manager state and the
|
||||
# persisted snapshot.
|
||||
response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {"defaultValue": "cli"},
|
||||
"unrelated_org_flag": {"defaultValue": True},
|
||||
"neko": {"defaultValue": False},
|
||||
"gbdemo-checkout-layout": {"defaultValue": "dev"},
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(_attrs())
|
||||
|
||||
snapshot = manager.export_state()
|
||||
assert snapshot is not None
|
||||
assert set(snapshot.features.keys()) == {ExperimentName.SYSTEM_PROMPT.value}
|
||||
|
||||
|
||||
def test_hydrate_drops_unknown_features() -> None:
|
||||
# When resuming a session saved by an older vibe version, the snapshot
|
||||
# may contain experiments that have since been retired. Filter them on
|
||||
# hydrate so the in-memory state matches the current ExperimentName.
|
||||
legacy_response = _response({
|
||||
ExperimentName.SYSTEM_PROMPT.value: {"defaultValue": "cli"},
|
||||
"retired_experiment": {"defaultValue": "control"},
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
manager.hydrate(legacy_response)
|
||||
|
||||
snapshot = manager.export_state()
|
||||
assert snapshot is not None
|
||||
assert set(snapshot.features.keys()) == {ExperimentName.SYSTEM_PROMPT.value}
|
||||
68
tests/core/experiments/test_models.py
Normal file
68
tests/core/experiments/test_models.py
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.models import EvalResponse, FeatureDefinition, FeatureRule
|
||||
|
||||
|
||||
class TestFeatureDefinition:
|
||||
def test_resolved_value_returns_force_when_present(self) -> None:
|
||||
feature = FeatureDefinition(
|
||||
defaultValue="cli", rules=[FeatureRule(force="cli_v2")]
|
||||
)
|
||||
assert feature.resolved_value() == "cli_v2"
|
||||
|
||||
def test_resolved_value_falls_back_to_default(self) -> None:
|
||||
feature = FeatureDefinition(defaultValue="cli")
|
||||
assert feature.resolved_value() == "cli"
|
||||
|
||||
def test_resolved_value_picks_first_rule_with_force(self) -> None:
|
||||
feature = FeatureDefinition(
|
||||
defaultValue="cli",
|
||||
rules=[FeatureRule(force=None), FeatureRule(force="cli_v2")],
|
||||
)
|
||||
assert feature.resolved_value() == "cli_v2"
|
||||
|
||||
|
||||
class TestEvalResponse:
|
||||
def test_parses_real_proxy_payload(self) -> None:
|
||||
# A reduced version of the live response captured during dev probing.
|
||||
sp_key = ExperimentName.SYSTEM_PROMPT.value
|
||||
raw = {
|
||||
"features": {
|
||||
"rbac_enabled": {"defaultValue": True},
|
||||
sp_key: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": "cli_v2",
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {
|
||||
"key": sp_key,
|
||||
"variations": [{}, "cli_v2"],
|
||||
},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"value": "cli_v2",
|
||||
"inExperiment": True,
|
||||
"hashAttribute": "id",
|
||||
"hashValue": "abc",
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
"experiments": [],
|
||||
"dateUpdated": "2026-05-11T13:29:48.394Z",
|
||||
}
|
||||
response = EvalResponse.model_validate(raw)
|
||||
assert response.features["rbac_enabled"].defaultValue is True
|
||||
sp = response.features[sp_key]
|
||||
assert sp.resolved_value() == "cli_v2"
|
||||
track = sp.rules[0].tracks[0]
|
||||
assert track.experiment.key == sp_key
|
||||
assert track.result.variationId == 1
|
||||
assert track.result.inExperiment is True
|
||||
168
tests/core/experiments/test_resume.py
Normal file
168
tests/core/experiments/test_resume.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_vibe_config
|
||||
from vibe.core.agents import AgentManager
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
from vibe.core.session.session_loader import SessionLoader
|
||||
from vibe.core.skills.manager import SkillManager
|
||||
from vibe.core.system_prompt import get_universal_system_prompt
|
||||
from vibe.core.tools.manager import ToolManager
|
||||
from vibe.core.types import SessionMetadata
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
self.calls: list[ExperimentAttributes] = []
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
self.calls.append(attributes)
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _attrs() -> ExperimentAttributes:
|
||||
return ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
|
||||
|
||||
def _response_forcing(variant: str) -> EvalResponse:
|
||||
return EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [
|
||||
{
|
||||
"force": variant,
|
||||
"tracks": [
|
||||
{
|
||||
"experiment": {
|
||||
"key": ExperimentName.SYSTEM_PROMPT.value
|
||||
},
|
||||
"result": {
|
||||
"key": "1",
|
||||
"variationId": 1,
|
||||
"inExperiment": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resume_round_trip_preserves_variant() -> None:
|
||||
response = _response_forcing("explore")
|
||||
fresh = ExperimentManager(client=_StubClient(response))
|
||||
await fresh.initialize(_attrs())
|
||||
|
||||
persisted_state = fresh.export_state()
|
||||
assert persisted_state is not None
|
||||
|
||||
serialized = persisted_state.model_dump_json()
|
||||
rehydrated_response = EvalResponse.model_validate_json(serialized)
|
||||
|
||||
stub = _StubClient(None)
|
||||
resumed = ExperimentManager(client=stub)
|
||||
resumed.hydrate(rehydrated_response)
|
||||
|
||||
assert resumed.get_variant(ExperimentName.SYSTEM_PROMPT) == "explore"
|
||||
assert resumed.assignments() == fresh.assignments()
|
||||
assert stub.calls == []
|
||||
|
||||
|
||||
def test_resume_old_session_without_experiments_field_falls_back_to_defaults(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
legacy_metadata = {
|
||||
"session_id": "legacy",
|
||||
"start_time": "2025-01-01T00:00:00+00:00",
|
||||
"end_time": None,
|
||||
"git_commit": None,
|
||||
"git_branch": None,
|
||||
"environment": {"working_directory": str(tmp_path)},
|
||||
"username": "tester",
|
||||
"loops": [],
|
||||
}
|
||||
meta_path = tmp_path / "meta.json"
|
||||
meta_path.write_text(json.dumps(legacy_metadata))
|
||||
|
||||
metadata = SessionLoader.load_metadata(tmp_path)
|
||||
|
||||
assert metadata.experiments is None
|
||||
|
||||
|
||||
def test_resume_overrides_still_win_over_hydrated_state() -> None:
|
||||
response = _response_forcing("explore")
|
||||
manager = ExperimentManager(
|
||||
client=_StubClient(None), overrides={ExperimentName.SYSTEM_PROMPT.value: "lean"}
|
||||
)
|
||||
manager.hydrate(response)
|
||||
|
||||
assert manager.get_variant(ExperimentName.SYSTEM_PROMPT) == "lean"
|
||||
|
||||
|
||||
def test_session_metadata_round_trips_experiments_field() -> None:
|
||||
response = _response_forcing("explore")
|
||||
metadata = SessionMetadata(
|
||||
session_id="s",
|
||||
start_time="2025-01-01T00:00:00+00:00",
|
||||
end_time=None,
|
||||
git_commit=None,
|
||||
git_branch=None,
|
||||
environment={"working_directory": "/tmp"},
|
||||
username="tester",
|
||||
experiments=response,
|
||||
)
|
||||
|
||||
serialized = metadata.model_dump_json()
|
||||
restored = SessionMetadata.model_validate_json(serialized)
|
||||
|
||||
assert restored.experiments is not None
|
||||
assert restored.experiments == response
|
||||
|
||||
|
||||
def _build_managers(config):
|
||||
return (
|
||||
ToolManager(lambda: config),
|
||||
SkillManager(lambda: config),
|
||||
AgentManager(lambda: config),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_graduated_experiment_with_deleted_variant_file_falls_back() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = _response_forcing("removed_after_graduation_2025_07")
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
manager.hydrate(response)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
|
||||
default_prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=None
|
||||
)
|
||||
assert prompt == default_prompt
|
||||
208
tests/core/experiments/test_session_helpers.py
Normal file
208
tests/core/experiments/test_session_helpers.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
from vibe.core.experiments.session import (
|
||||
hydrate_experiments_from_session,
|
||||
initialize_experiments,
|
||||
)
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _make_config(
|
||||
*, enable_telemetry: bool = True, enable_experiments: bool = True
|
||||
) -> Any:
|
||||
config = MagicMock()
|
||||
config.enable_telemetry = enable_telemetry
|
||||
config.experiments.enable = enable_experiments
|
||||
return config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_telemetry_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(enable_telemetry=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_experiments_disabled(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# `experiments.enable=False` must short-circuit even when telemetry is on
|
||||
# — it is the user's opt-out for being assigned to A/B tests without
|
||||
# disabling all telemetry.
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(enable_experiments=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_no_mistral_provider(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session.get_mistral_provider_and_api_key",
|
||||
lambda _config: None,
|
||||
)
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_false_when_remote_eval_fails(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
# Regression: even if telemetry is enabled and a Mistral key is set,
|
||||
# a failed remote eval (returns None) leaves manager state empty —
|
||||
# the helper must NOT report success, so the caller skips the
|
||||
# unnecessary system prompt refresh.
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session.get_mistral_provider_and_api_key",
|
||||
lambda _config: (MagicMock(), "fake-key"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session._build_attributes",
|
||||
lambda *_args, **_kwargs: ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
),
|
||||
)
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
persist.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialize_returns_true_and_persists_when_remote_eval_succeeds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session.get_mistral_provider_and_api_key",
|
||||
lambda _config: (MagicMock(), "fake-key"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vibe.core.experiments.session._build_attributes",
|
||||
lambda *_args, **_kwargs: ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
),
|
||||
)
|
||||
persist = AsyncMock()
|
||||
session_logger = MagicMock()
|
||||
session_logger.persist_experiments = persist
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
|
||||
result = await initialize_experiments(
|
||||
config=_make_config(),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
entrypoint_metadata=None,
|
||||
)
|
||||
|
||||
assert result is True
|
||||
persist.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hydrate_returns_false_when_telemetry_disabled() -> None:
|
||||
session_logger = MagicMock()
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
|
||||
})
|
||||
session_logger.session_metadata.experiments = response
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await hydrate_experiments_from_session(
|
||||
config=_make_config(enable_telemetry=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert manager.export_state() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hydrate_returns_false_when_experiments_disabled() -> None:
|
||||
# Regression: without this gate, a user who flipped experiments.enable
|
||||
# to False between sessions would still resume into a hydrated variant.
|
||||
session_logger = MagicMock()
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"vibe_cli_system_prompt": {"defaultValue": "cli"}}
|
||||
})
|
||||
session_logger.session_metadata.experiments = response
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
result = await hydrate_experiments_from_session(
|
||||
config=_make_config(enable_experiments=False),
|
||||
manager=manager,
|
||||
session_logger=session_logger,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
assert manager.export_state() is None
|
||||
186
tests/core/experiments/test_system_prompt_variant.py
Normal file
186
tests/core/experiments/test_system_prompt_variant.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import build_test_vibe_config
|
||||
from vibe.core.agents import AgentManager
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.experiments.client import RemoteEvalClient
|
||||
from vibe.core.experiments.manager import ExperimentManager
|
||||
from vibe.core.experiments.models import EvalResponse, ExperimentAttributes
|
||||
from vibe.core.prompts import load_system_prompt
|
||||
from vibe.core.skills.manager import SkillManager
|
||||
from vibe.core.system_prompt import get_universal_system_prompt
|
||||
from vibe.core.tools.manager import ToolManager
|
||||
|
||||
|
||||
class _StubClient(RemoteEvalClient):
|
||||
def __init__(self, response: EvalResponse | None) -> None:
|
||||
self._response = response
|
||||
|
||||
async def evaluate(self, attributes: ExperimentAttributes) -> EvalResponse | None:
|
||||
return self._response
|
||||
|
||||
async def aclose(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _build_managers(config):
|
||||
return (
|
||||
ToolManager(lambda: config),
|
||||
SkillManager(lambda: config),
|
||||
AgentManager(lambda: config),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_prompt_uses_assigned_variant() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "tests", "tracks": []}],
|
||||
}
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt.startswith("You are Vibe, a super useful programming assistant.")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_prompt_falls_back_to_default_when_variant_unknown() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "nonexistent_prompt", "tracks": []}],
|
||||
}
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
# Falls back to the default `cli` system prompt.
|
||||
assert "You are Mistral Vibe" in prompt or "You are Vibe" in prompt
|
||||
|
||||
|
||||
def test_system_prompt_uses_default_when_no_manager() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="cli",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager
|
||||
)
|
||||
assert "You are Mistral Vibe" in prompt or "You are Vibe" in prompt
|
||||
|
||||
|
||||
def test_system_prompt_honors_user_config_when_manager_uninitialized() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="lean",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
manager = ExperimentManager(client=_StubClient(None))
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt == load_system_prompt("lean")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_system_prompt_honors_user_config_when_no_remote_assignment() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="lean",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {"some_other_feature": {"defaultValue": True}}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt == load_system_prompt("lean")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_config_overrides_assigned_experiment_variant() -> None:
|
||||
config = build_test_vibe_config(
|
||||
system_prompt_id="lean",
|
||||
include_project_context=False,
|
||||
include_prompt_detail=False,
|
||||
include_model_info=False,
|
||||
include_commit_signature=False,
|
||||
)
|
||||
response = EvalResponse.model_validate({
|
||||
"features": {
|
||||
ExperimentName.SYSTEM_PROMPT.value: {
|
||||
"defaultValue": "cli",
|
||||
"rules": [{"force": "explore", "tracks": []}],
|
||||
}
|
||||
}
|
||||
})
|
||||
manager = ExperimentManager(client=_StubClient(response))
|
||||
await manager.initialize(
|
||||
ExperimentAttributes(
|
||||
userId="x", entrypoint="cli", agent_version="0", os="darwin"
|
||||
)
|
||||
)
|
||||
|
||||
tool_manager, skill_manager, agent_manager = _build_managers(config)
|
||||
prompt = get_universal_system_prompt(
|
||||
tool_manager, config, skill_manager, agent_manager, experiment_manager=manager
|
||||
)
|
||||
assert prompt == load_system_prompt("lean")
|
||||
assert prompt != load_system_prompt("explore")
|
||||
31
tests/core/experiments/test_telemetry_integration.py
Normal file
31
tests/core/experiments/test_telemetry_integration.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from tests.conftest import build_test_vibe_config
|
||||
from vibe.core.experiments.active import ExperimentName
|
||||
from vibe.core.telemetry.send import TelemetryClient
|
||||
|
||||
|
||||
def test_build_client_event_metadata_omits_experiments_when_empty() -> None:
|
||||
config = build_test_vibe_config(enable_telemetry=True)
|
||||
client = TelemetryClient(
|
||||
config_getter=lambda: config, experiments_getter=lambda: {}
|
||||
)
|
||||
metadata = client.build_client_event_metadata()
|
||||
assert "experiments" not in metadata
|
||||
|
||||
|
||||
def test_build_client_event_metadata_includes_experiments_when_present() -> None:
|
||||
config = build_test_vibe_config(enable_telemetry=True)
|
||||
sp_key = ExperimentName.SYSTEM_PROMPT.value
|
||||
client = TelemetryClient(
|
||||
config_getter=lambda: config, experiments_getter=lambda: {sp_key: "1"}
|
||||
)
|
||||
metadata = client.build_client_event_metadata()
|
||||
assert metadata["experiments"] == {sp_key: "1"}
|
||||
|
||||
|
||||
def test_build_client_event_metadata_works_without_getter() -> None:
|
||||
config = build_test_vibe_config(enable_telemetry=True)
|
||||
client = TelemetryClient(config_getter=lambda: config)
|
||||
metadata = client.build_client_event_metadata()
|
||||
assert "experiments" not in metadata
|
||||
123
tests/core/session/test_last_session_pointer.py
Normal file
123
tests/core/session/test_last_session_pointer.py
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vibe.core.config import SessionLoggingConfig
|
||||
from vibe.core.session import last_session_pointer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session_logging(tmp_path: Path) -> SessionLoggingConfig:
|
||||
return SessionLoggingConfig(save_dir=str(tmp_path))
|
||||
|
||||
|
||||
def _set_tty(monkeypatch: pytest.MonkeyPatch, key: str | None) -> None:
|
||||
monkeypatch.setattr(last_session_pointer, "current_tty_key", lambda: key)
|
||||
|
||||
|
||||
def test_load_returns_none_when_no_tty(
|
||||
session_logging: SessionLoggingConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_set_tty(monkeypatch, None)
|
||||
assert last_session_pointer.load(session_logging) is None
|
||||
|
||||
|
||||
def test_load_returns_none_when_no_pointer_written(
|
||||
session_logging: SessionLoggingConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_set_tty(monkeypatch, "ttys001")
|
||||
assert last_session_pointer.load(session_logging) is None
|
||||
|
||||
|
||||
def test_record_then_load_round_trip(
|
||||
session_logging: SessionLoggingConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_set_tty(monkeypatch, "ttys001")
|
||||
last_session_pointer.record(session_logging, "abc-123")
|
||||
assert last_session_pointer.load(session_logging) == "abc-123"
|
||||
|
||||
|
||||
def test_record_skips_when_no_tty(
|
||||
session_logging: SessionLoggingConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_set_tty(monkeypatch, None)
|
||||
last_session_pointer.record(session_logging, "abc-123")
|
||||
pointer_dir = Path(session_logging.save_dir) / last_session_pointer.POINTER_DIR_NAME
|
||||
assert not pointer_dir.exists()
|
||||
|
||||
|
||||
def test_record_skips_when_logging_disabled(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
disabled = SessionLoggingConfig(save_dir=str(tmp_path), enabled=False)
|
||||
_set_tty(monkeypatch, "ttys001")
|
||||
last_session_pointer.record(disabled, "abc-123")
|
||||
assert not (tmp_path / last_session_pointer.POINTER_DIR_NAME).exists()
|
||||
|
||||
|
||||
def test_pointers_are_per_tty(
|
||||
session_logging: SessionLoggingConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_set_tty(monkeypatch, "ttys001")
|
||||
last_session_pointer.record(session_logging, "session-a")
|
||||
|
||||
_set_tty(monkeypatch, "ttys002")
|
||||
last_session_pointer.record(session_logging, "session-b")
|
||||
assert last_session_pointer.load(session_logging) == "session-b"
|
||||
|
||||
_set_tty(monkeypatch, "ttys001")
|
||||
assert last_session_pointer.load(session_logging) == "session-a"
|
||||
|
||||
|
||||
def test_record_ignores_empty_session_id(
|
||||
session_logging: SessionLoggingConfig, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
_set_tty(monkeypatch, "ttys001")
|
||||
last_session_pointer.record(session_logging, None)
|
||||
last_session_pointer.record(session_logging, "")
|
||||
assert last_session_pointer.load(session_logging) is None
|
||||
|
||||
|
||||
def test_current_tty_key_returns_none_when_ttyname_is_unavailable(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delattr(last_session_pointer.os, "ttyname", raising=False)
|
||||
|
||||
assert last_session_pointer.current_tty_key() is None
|
||||
|
||||
|
||||
def _patch_windows(monkeypatch: pytest.MonkeyPatch, hwnd: int) -> None:
|
||||
import ctypes
|
||||
from types import SimpleNamespace
|
||||
|
||||
fake_windll = SimpleNamespace(
|
||||
kernel32=SimpleNamespace(GetConsoleWindow=lambda: hwnd)
|
||||
)
|
||||
monkeypatch.setattr(last_session_pointer.sys, "platform", "win32")
|
||||
monkeypatch.setattr(ctypes, "windll", fake_windll, raising=False)
|
||||
|
||||
|
||||
def test_current_tty_key_uses_console_hwnd_on_windows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_patch_windows(monkeypatch, hwnd=12345)
|
||||
assert last_session_pointer.current_tty_key() == "conhost-12345"
|
||||
|
||||
|
||||
def test_current_tty_key_falls_back_to_wt_session(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_patch_windows(monkeypatch, hwnd=0)
|
||||
monkeypatch.setenv("WT_SESSION", "abcd-1234")
|
||||
assert last_session_pointer.current_tty_key() == "wt-abcd-1234"
|
||||
|
||||
|
||||
def test_current_tty_key_falls_back_to_ppid_on_windows(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_patch_windows(monkeypatch, hwnd=0)
|
||||
monkeypatch.delenv("WT_SESSION", raising=False)
|
||||
monkeypatch.setattr(last_session_pointer.os, "getppid", lambda: 4242)
|
||||
assert last_session_pointer.current_tty_key() == "ppid-4242"
|
||||
389
tests/core/test_add_dir.py
Normal file
389
tests/core/test_add_dir.py
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vibe.core.config.harness_files import (
|
||||
HarnessFilesManager,
|
||||
get_harness_files_manager,
|
||||
init_harness_files_manager,
|
||||
reset_harness_files_manager,
|
||||
)
|
||||
from vibe.core.paths import AGENTS_MD_FILENAME
|
||||
from vibe.core.tools.utils import is_path_within_workdir
|
||||
from vibe.core.trusted_folders import trusted_folders_manager
|
||||
|
||||
|
||||
class TestHarnessFilesManagerAdditionalDirs:
|
||||
def test_default_additional_dirs_empty(self) -> None:
|
||||
mgr = HarnessFilesManager(sources=("user",))
|
||||
assert mgr._additional_dirs == ()
|
||||
|
||||
def test_init_with_additional_dirs(self, tmp_path: Path) -> None:
|
||||
d1 = tmp_path / "extra1"
|
||||
d1.mkdir()
|
||||
d2 = tmp_path / "extra2"
|
||||
d2.mkdir()
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project", additional_dirs=[d1, d2])
|
||||
mgr = get_harness_files_manager()
|
||||
# init normalizes to resolved paths so the symlink/relative-vs-absolute
|
||||
# spellings of the same dir compare equal on reinit.
|
||||
assert mgr._additional_dirs == (d1.resolve(), d2.resolve())
|
||||
|
||||
def test_reinit_idempotent_when_paths_resolve_equal(self, tmp_path: Path) -> None:
|
||||
target = tmp_path / "target"
|
||||
target.mkdir()
|
||||
link = tmp_path / "link"
|
||||
link.symlink_to(target, target_is_directory=True)
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project", additional_dirs=[target])
|
||||
# Second call with the symlink spelling must NOT raise — both resolve
|
||||
# to the same dir.
|
||||
init_harness_files_manager("user", "project", additional_dirs=[link])
|
||||
|
||||
def test_init_without_additional_dirs(self) -> None:
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project")
|
||||
mgr = get_harness_files_manager()
|
||||
assert mgr._additional_dirs == ()
|
||||
|
||||
def test_reinit_with_same_additional_dirs_is_idempotent(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
d = tmp_path / "extra"
|
||||
d.mkdir()
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project", additional_dirs=[d])
|
||||
# Same sources + same additional_dirs: no-op, no raise.
|
||||
init_harness_files_manager("user", "project", additional_dirs=[d])
|
||||
|
||||
def test_reinit_with_different_additional_dirs_raises(self, tmp_path: Path) -> None:
|
||||
d1 = tmp_path / "extra1"
|
||||
d1.mkdir()
|
||||
d2 = tmp_path / "extra2"
|
||||
d2.mkdir()
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project", additional_dirs=[d1])
|
||||
with pytest.raises(RuntimeError, match="different configuration"):
|
||||
init_harness_files_manager("user", "project", additional_dirs=[d2])
|
||||
|
||||
|
||||
class TestAdditionalDirsDiscovery:
|
||||
def test_discovers_tools_from_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
tools_dir = extra / ".vibe" / "tools"
|
||||
tools_dir.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert tools_dir in mgr.project_tools_dirs
|
||||
|
||||
def test_discovers_skills_from_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
skills_dir = extra / ".vibe" / "skills"
|
||||
skills_dir.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert skills_dir in mgr.project_skills_dirs
|
||||
|
||||
def test_discovers_agents_from_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
agents_dir = extra / ".vibe" / "agents"
|
||||
agents_dir.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert agents_dir in mgr.project_agents_dirs
|
||||
|
||||
def test_discovers_agents_skills_from_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
skills_dir = extra / ".agents" / "skills"
|
||||
skills_dir.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert skills_dir in mgr.project_skills_dirs
|
||||
|
||||
def test_discovers_prompts_from_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
prompts_dir = extra / ".vibe" / "prompts"
|
||||
prompts_dir.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert prompts_dir in mgr.project_prompts_dirs
|
||||
|
||||
def test_no_dirs_when_additional_dir_has_no_vibe(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "bare_project"
|
||||
extra.mkdir()
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert mgr.project_tools_dirs == []
|
||||
assert mgr.project_skills_dirs == []
|
||||
assert mgr.project_agents_dirs == []
|
||||
|
||||
|
||||
class TestAdditionalDirsAgentsMd:
|
||||
def test_load_project_docs_includes_additional_dir_agents_md(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
extra.mkdir()
|
||||
agents_md = extra / AGENTS_MD_FILENAME
|
||||
agents_md.write_text("Extra project instructions", encoding="utf-8")
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
docs = mgr.load_project_docs()
|
||||
assert any(d == extra and "Extra project" in content for d, content in docs)
|
||||
|
||||
def test_load_project_docs_skips_missing_agents_md(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "no_agents_md"
|
||||
extra.mkdir()
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
docs = mgr.load_project_docs()
|
||||
assert not any(d == extra for d, _ in docs)
|
||||
|
||||
def test_load_project_docs_skips_empty_agents_md(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "empty_agents_md"
|
||||
extra.mkdir()
|
||||
(extra / AGENTS_MD_FILENAME).write_text(" \n", encoding="utf-8")
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
docs = mgr.load_project_docs()
|
||||
assert not any(d == extra for d, _ in docs)
|
||||
|
||||
def test_find_subdirectory_agents_md_in_additional_dir(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
sub = extra / "sub"
|
||||
sub.mkdir(parents=True)
|
||||
agents_md = sub / AGENTS_MD_FILENAME
|
||||
agents_md.write_text("Sub instructions", encoding="utf-8")
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
file_in_sub = sub / "main.py"
|
||||
docs = mgr.find_subdirectory_agents_md(file_in_sub)
|
||||
assert any("Sub instructions" in content for _, content in docs)
|
||||
|
||||
def test_find_subdirectory_agents_md_resolves_additional_dir(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
extra = tmp_path / "extra_project"
|
||||
sub = extra / "sub"
|
||||
sub.mkdir(parents=True)
|
||||
link = tmp_path / "extra_link"
|
||||
link.symlink_to(extra, target_is_directory=True)
|
||||
agents_md = sub / AGENTS_MD_FILENAME
|
||||
agents_md.write_text("Sub instructions", encoding="utf-8")
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(link,))
|
||||
docs = mgr.find_subdirectory_agents_md(sub / "main.py")
|
||||
assert any("Sub instructions" in content for _, content in docs)
|
||||
|
||||
def test_find_subdirectory_agents_md_outside_all_dirs(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra"
|
||||
extra.mkdir()
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
docs = mgr.find_subdirectory_agents_md(outside / "file.py")
|
||||
assert docs == []
|
||||
|
||||
|
||||
class TestFilePermissionsAdditionalDirs:
|
||||
def test_is_within_workdir_for_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra"
|
||||
extra.mkdir()
|
||||
file_in_extra = extra / "some_file.py"
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project", additional_dirs=[extra])
|
||||
|
||||
assert is_path_within_workdir(str(file_in_extra))
|
||||
|
||||
def test_is_within_workdir_resolves_additional_dir(self, tmp_path: Path) -> None:
|
||||
extra = tmp_path / "extra"
|
||||
extra.mkdir()
|
||||
link = tmp_path / "extra_link"
|
||||
link.symlink_to(extra, target_is_directory=True)
|
||||
file_in_extra = extra / "some_file.py"
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project", additional_dirs=[link])
|
||||
|
||||
assert is_path_within_workdir(str(file_in_extra))
|
||||
|
||||
def test_is_within_workdir_for_cwd(self, tmp_working_directory: Path) -> None:
|
||||
file_in_cwd = tmp_working_directory / "file.py"
|
||||
assert is_path_within_workdir(str(file_in_cwd))
|
||||
|
||||
def test_is_not_within_workdir_for_outside_path(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
outside = tmp_path / "outside"
|
||||
outside.mkdir()
|
||||
cwd = tmp_path / "cwd"
|
||||
cwd.mkdir()
|
||||
monkeypatch.chdir(cwd)
|
||||
|
||||
reset_harness_files_manager()
|
||||
init_harness_files_manager("user", "project")
|
||||
|
||||
assert not is_path_within_workdir(str(outside / "file.py"))
|
||||
|
||||
def test_is_path_within_workdir_when_manager_uninitialized(
|
||||
self, tmp_working_directory: Path
|
||||
) -> None:
|
||||
reset_harness_files_manager()
|
||||
# Without a manager, only cwd counts.
|
||||
assert is_path_within_workdir(str(tmp_working_directory / "file.py"))
|
||||
|
||||
|
||||
class TestAdditionalDirsAreImplicitlyTrusted:
|
||||
def test_load_project_docs_does_not_require_trust_for_additional_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# Untrusted cwd: primary workdir contributes nothing.
|
||||
monkeypatch.setattr(trusted_folders_manager, "is_trusted", lambda _: False)
|
||||
|
||||
extra = tmp_path / "extra_project"
|
||||
extra.mkdir()
|
||||
(extra / AGENTS_MD_FILENAME).write_text(
|
||||
"Trusted via --add-dir", encoding="utf-8"
|
||||
)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(extra,)
|
||||
)
|
||||
docs = mgr.load_project_docs()
|
||||
assert any("Trusted via --add-dir" in content for _, content in docs)
|
||||
|
||||
def test_project_tools_dirs_skip_trust_check_for_additional_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(trusted_folders_manager, "is_trusted", lambda _: False)
|
||||
|
||||
extra = tmp_path / "extra_project"
|
||||
tools_dir = extra / ".vibe" / "tools"
|
||||
tools_dir.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(extra,)
|
||||
)
|
||||
assert tools_dir in mgr.project_tools_dirs
|
||||
|
||||
|
||||
class TestLoadProjectDocsDedupe:
|
||||
def test_dedupes_when_cwd_under_additional_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
extra = (tmp_path / "extra_project").resolve()
|
||||
extra.mkdir()
|
||||
sub = extra / "sub"
|
||||
sub.mkdir()
|
||||
(extra / AGENTS_MD_FILENAME).write_text("Root instructions", encoding="utf-8")
|
||||
(sub / AGENTS_MD_FILENAME).write_text("Sub instructions", encoding="utf-8")
|
||||
monkeypatch.chdir(sub)
|
||||
|
||||
trusted_folders_manager.trust_for_session(extra)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(extra,)
|
||||
)
|
||||
docs = mgr.load_project_docs()
|
||||
|
||||
emitted_dirs = [d for d, _ in docs]
|
||||
assert emitted_dirs.count(extra) == 1
|
||||
assert sub.resolve() in emitted_dirs
|
||||
|
||||
|
||||
class TestHookFilesAdditionalDirs:
|
||||
def test_hook_files_includes_additional_dir_hooks_toml(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
extra = tmp_path / "extra"
|
||||
extra.mkdir()
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(extra,))
|
||||
assert extra / ".vibe" / "hooks.toml" in mgr.hook_files
|
||||
|
||||
def test_hook_files_deduplicates_workdir_and_additional_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workdir = tmp_path / "project"
|
||||
workdir.mkdir()
|
||||
monkeypatch.chdir(workdir)
|
||||
trusted_folders_manager.trust_for_session(workdir)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(workdir.resolve(),)
|
||||
)
|
||||
hook_file = workdir / ".vibe" / "hooks.toml"
|
||||
|
||||
assert mgr.hook_files.count(hook_file.resolve()) == 1
|
||||
|
||||
|
||||
class TestProjectPromptsDirsAdditionalDirs:
|
||||
def test_project_prompts_dirs_deduplicates_workdir_and_additional_dir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
workdir = tmp_path / "project"
|
||||
(workdir / ".vibe" / "prompts").mkdir(parents=True)
|
||||
monkeypatch.chdir(workdir)
|
||||
trusted_folders_manager.trust_for_session(workdir)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(workdir.resolve(),)
|
||||
)
|
||||
|
||||
assert len(mgr.project_prompts_dirs) == 1
|
||||
|
||||
|
||||
class TestProjectRootsNestedDedup:
|
||||
def test_nested_add_dirs_collapse(self, tmp_path: Path) -> None:
|
||||
outer = (tmp_path / "outer").resolve()
|
||||
inner = outer / "inner"
|
||||
inner.mkdir(parents=True)
|
||||
|
||||
mgr = HarnessFilesManager(sources=("user",), _additional_dirs=(outer, inner))
|
||||
assert mgr.project_roots == [outer]
|
||||
|
||||
def test_add_dir_containing_cwd_keeps_both(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
outer = (tmp_path / "outer").resolve()
|
||||
inner = outer / "inner"
|
||||
inner.mkdir(parents=True)
|
||||
(outer / AGENTS_MD_FILENAME).write_text("Outer instructions", encoding="utf-8")
|
||||
(inner / AGENTS_MD_FILENAME).write_text("Inner instructions", encoding="utf-8")
|
||||
monkeypatch.chdir(inner)
|
||||
trusted_folders_manager.trust_for_session(inner)
|
||||
trusted_folders_manager.trust_for_session(outer)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(outer,)
|
||||
)
|
||||
# cwd survives so its walk-up semantics still work; the add-dir keeps
|
||||
# its own root-level discovery.
|
||||
assert mgr.project_roots == [inner, outer]
|
||||
docs = mgr.load_project_docs()
|
||||
assert any("Outer" in c for _, c in docs)
|
||||
assert any("Inner" in c for _, c in docs)
|
||||
|
||||
def test_add_dir_nested_under_trusted_workdir(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
outer = (tmp_path / "outer").resolve()
|
||||
inner = outer / "inner"
|
||||
inner.mkdir(parents=True)
|
||||
monkeypatch.chdir(outer)
|
||||
trusted_folders_manager.trust_for_session(outer)
|
||||
|
||||
mgr = HarnessFilesManager(
|
||||
sources=("user", "project"), _additional_dirs=(inner,)
|
||||
)
|
||||
assert mgr.project_roots == [outer]
|
||||
|
|
@ -14,7 +14,7 @@ class TestTrustedWorkdir:
|
|||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
mgr = HarnessFilesManager(sources=("user",))
|
||||
assert mgr.trusted_workdir is None
|
||||
assert mgr._trusted_workdir is None
|
||||
|
||||
def test_returns_none_when_not_trusted(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
@ -22,7 +22,7 @@ class TestTrustedWorkdir:
|
|||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(trusted_folders_manager, "is_trusted", lambda _: False)
|
||||
mgr = HarnessFilesManager(sources=("user", "project"))
|
||||
assert mgr.trusted_workdir is None
|
||||
assert mgr._trusted_workdir is None
|
||||
|
||||
def test_returns_cwd_when_project_in_sources_and_trusted(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
|
|
@ -30,7 +30,7 @@ class TestTrustedWorkdir:
|
|||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(trusted_folders_manager, "is_trusted", lambda _: True)
|
||||
mgr = HarnessFilesManager(sources=("user", "project"))
|
||||
assert mgr.trusted_workdir == tmp_path
|
||||
assert mgr._trusted_workdir == tmp_path
|
||||
|
||||
|
||||
class TestProjectToolsDirs:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from pathlib import Path
|
|||
from vibe.core.paths._local_config_walk import (
|
||||
_MAX_DIRS,
|
||||
WALK_MAX_DEPTH,
|
||||
ConfigWalkResult,
|
||||
walk_local_config_dirs,
|
||||
)
|
||||
|
||||
|
|
@ -166,3 +167,29 @@ class TestWalkConfigDirs:
|
|||
assert resolved / ".vibe" in result.config_dirs
|
||||
assert resolved / ".agents" in result.config_dirs
|
||||
assert resolved / "sub" / ".vibe" in result.config_dirs
|
||||
|
||||
|
||||
class TestConfigWalkResultOr:
|
||||
def test_or_concatenates_each_field(self) -> None:
|
||||
a = ConfigWalkResult(
|
||||
config_dirs=(Path("/a/.vibe"),),
|
||||
tools=(Path("/a/.vibe/tools"),),
|
||||
skills=(Path("/a/.vibe/skills"),),
|
||||
agents=(Path("/a/.vibe/agents"),),
|
||||
)
|
||||
b = ConfigWalkResult(
|
||||
config_dirs=(Path("/b/.vibe"),),
|
||||
tools=(Path("/b/.vibe/tools"),),
|
||||
skills=(Path("/b/.vibe/skills"),),
|
||||
agents=(Path("/b/.vibe/agents"),),
|
||||
)
|
||||
merged = a | b
|
||||
assert merged.config_dirs == (Path("/a/.vibe"), Path("/b/.vibe"))
|
||||
assert merged.tools == (Path("/a/.vibe/tools"), Path("/b/.vibe/tools"))
|
||||
assert merged.skills == (Path("/a/.vibe/skills"), Path("/b/.vibe/skills"))
|
||||
assert merged.agents == (Path("/a/.vibe/agents"), Path("/b/.vibe/agents"))
|
||||
|
||||
def test_or_with_empty_is_identity(self) -> None:
|
||||
a = ConfigWalkResult(tools=(Path("/a/.vibe/tools"),))
|
||||
assert (a | ConfigWalkResult()) == a
|
||||
assert (ConfigWalkResult() | a) == a
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ def _make_manager(
|
|||
async def save_messages() -> None:
|
||||
save_calls.append(True)
|
||||
|
||||
def reset_session() -> None:
|
||||
async def reset_session() -> None:
|
||||
reset_calls.append(True)
|
||||
|
||||
mgr = RewindManager(
|
||||
|
|
|
|||
|
|
@ -102,6 +102,79 @@ class TestReadSafe:
|
|||
read_safe(tmp_path / "nope.txt")
|
||||
|
||||
|
||||
class TestReadSafeNewlines:
|
||||
def test_lf(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "lf.txt"
|
||||
f.write_bytes(b"a\nb\nc\n")
|
||||
got = read_safe(f)
|
||||
assert got.text == "a\nb\nc\n"
|
||||
assert got.newline == "\n"
|
||||
|
||||
def test_crlf(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "crlf.txt"
|
||||
f.write_bytes(b"a\r\nb\r\nc\r\n")
|
||||
got = read_safe(f)
|
||||
assert got.text == "a\nb\nc\n"
|
||||
assert got.newline == "\r\n"
|
||||
|
||||
def test_cr(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "cr.txt"
|
||||
f.write_bytes(b"a\rb\rc\r")
|
||||
got = read_safe(f)
|
||||
assert got.text == "a\nb\nc\n"
|
||||
assert got.newline == "\r"
|
||||
|
||||
def test_mixed_picks_most_frequent(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "mixed.txt"
|
||||
f.write_bytes(b"a\r\nb\r\nc\rd\n")
|
||||
got = read_safe(f)
|
||||
assert got.text == "a\nb\nc\nd\n"
|
||||
assert got.newline == "\r\n"
|
||||
|
||||
@pytest.mark.parametrize(("linesep", "expected"), [("\n", "\n"), ("\r\n", "\r\n")])
|
||||
def test_no_newline_defaults_to_os_linesep(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
linesep: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(io_utils.os, "linesep", linesep)
|
||||
f = tmp_path / "single.txt"
|
||||
f.write_bytes(b"hello")
|
||||
got = read_safe(f)
|
||||
assert got.text == "hello"
|
||||
assert got.newline == expected
|
||||
|
||||
@pytest.mark.parametrize(("linesep", "expected"), [("\n", "\n"), ("\r\n", "\r\n")])
|
||||
def test_empty_defaults_to_os_linesep(
|
||||
self,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
linesep: str,
|
||||
expected: str,
|
||||
) -> None:
|
||||
monkeypatch.setattr(io_utils.os, "linesep", linesep)
|
||||
f = tmp_path / "empty.txt"
|
||||
f.write_bytes(b"")
|
||||
got = read_safe(f)
|
||||
assert got.text == ""
|
||||
assert got.newline == expected
|
||||
|
||||
def test_decode_safe_reports_newline(self) -> None:
|
||||
got = decode_safe(b"a\r\nb\r\n")
|
||||
assert got.text == "a\nb\n"
|
||||
assert got.newline == "\r\n"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_reports_newline(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "crlf.txt"
|
||||
f.write_bytes(b"a\r\nb\r\n")
|
||||
got = await read_safe_async(f)
|
||||
assert got.text == "a\nb\n"
|
||||
assert got.newline == "\r\n"
|
||||
|
||||
|
||||
class TestReadSafeResultEncoding:
|
||||
def test_reports_utf8_for_plain_utf8_file(self, tmp_path: Path) -> None:
|
||||
f = tmp_path / "x.txt"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue