Co-authored-by: Clément Drouin <clement.drouin@mistral.ai>
Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai>
Co-authored-by: Gauthier Guinet <43207538+Gguinet@users.noreply.github.com>
Co-authored-by: Kim-Adeline Miguel <kimadeline.miguel@mistral.ai>
Co-authored-by: Michel Thomazo <51709227+michelTho@users.noreply.github.com>
Co-authored-by: Quentin <torroba.q@gmail.com>
Co-authored-by: Simon <80467011+sorgfresser@users.noreply.github.com>
Co-authored-by: Simon Van de Kerckhove <simon.vandekerckhove@mistral.ai>
Co-authored-by: Vincent G <10739306+VinceOPS@users.noreply.github.com>
Co-authored-by: angelapopopo <angele.lenglemetz@mistral.ai>
Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Mathias Gesbert 2026-03-23 18:45:21 +01:00 committed by GitHub
parent 5103019b01
commit eb580209d4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
180 changed files with 11136 additions and 1030 deletions

54
tests/tools/test_arity.py Normal file
View file

@ -0,0 +1,54 @@
from __future__ import annotations
from vibe.core.tools.arity import build_session_pattern
class TestBuildSessionPattern:
def test_single_token_in_arity(self):
assert build_session_pattern(["mkdir", "foo"]) == "mkdir *"
def test_single_token_not_in_arity(self):
assert build_session_pattern(["whoami"]) == "whoami *"
def test_two_token_arity(self):
assert build_session_pattern(["git", "checkout", "main"]) == "git checkout *"
def test_three_token_arity(self):
assert build_session_pattern(["npm", "run", "dev"]) == "npm run dev *"
def test_longer_prefix_wins(self):
# "git" is arity 2, but "git stash" is arity 3
assert build_session_pattern(["git", "stash", "pop"]) == "git stash pop *"
def test_docker_compose(self):
assert (
build_session_pattern(["docker", "compose", "up", "-d"])
== "docker compose up *"
)
def test_empty_tokens(self):
assert build_session_pattern([]) == ""
def test_unknown_command_returns_first_token(self):
assert build_session_pattern(["mycommand", "arg1", "arg2"]) == "mycommand *"
def test_cat_is_arity_1(self):
assert build_session_pattern(["cat", "file.txt"]) == "cat *"
def test_rm_is_arity_1(self):
assert build_session_pattern(["rm", "-rf", "dir"]) == "rm *"
def test_uv_run(self):
assert build_session_pattern(["uv", "run", "pytest"]) == "uv run pytest *"
def test_pip_install(self):
assert build_session_pattern(["pip", "install", "numpy"]) == "pip install *"
def test_git_remote_add(self):
assert (
build_session_pattern(["git", "remote", "add", "origin", "url"])
== "git remote add *"
)
def test_gh_pr_list(self):
assert build_session_pattern(["gh", "pr", "list"]) == "gh pr list *"

View file

@ -5,6 +5,7 @@ import pytest
from tests.mock.utils import collect_result
from vibe.core.tools.base import BaseToolState, ToolError, ToolPermission
from vibe.core.tools.builtins.bash import Bash, BashArgs, BashToolConfig
from vibe.core.tools.permissions import PermissionContext
@pytest.fixture
@ -80,7 +81,10 @@ def test_find_not_in_default_allowlist():
bash_tool = Bash(config=BashToolConfig(), state=BaseToolState())
# find -exec runs arbitrary commands; must not be allowlisted by default
permission = bash_tool.resolve_permission(BashArgs(command="find . -exec id \\;"))
assert permission is not ToolPermission.ALWAYS
assert (
not isinstance(permission, PermissionContext)
or permission.permission is not ToolPermission.ALWAYS
)
def test_resolve_permission():
@ -92,9 +96,13 @@ def test_resolve_permission():
mixed = bash_tool.resolve_permission(BashArgs(command="pwd && whoami"))
empty = bash_tool.resolve_permission(BashArgs(command=""))
assert allowlisted is ToolPermission.ALWAYS
assert denylisted is ToolPermission.NEVER
assert mixed is None
assert isinstance(allowlisted, PermissionContext)
assert allowlisted.permission is ToolPermission.ALWAYS
assert isinstance(denylisted, PermissionContext)
assert denylisted.permission is ToolPermission.NEVER
assert isinstance(mixed, PermissionContext)
assert mixed.permission is ToolPermission.ASK
assert any(rp.label == "whoami *" for rp in mixed.required_permissions)
assert empty is None
@ -108,80 +116,95 @@ class TestResolvePermissionWindowsSyntax:
def test_dir_with_windows_flags_allowlisted(self):
bash_tool = self._make_bash(allowlist=["dir"])
result = bash_tool.resolve_permission(BashArgs(command="dir /s /b"))
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_type_command_allowlisted(self):
bash_tool = self._make_bash(allowlist=["type"])
result = bash_tool.resolve_permission(BashArgs(command="type file.txt"))
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_findstr_allowlisted(self):
bash_tool = self._make_bash(allowlist=["findstr"])
result = bash_tool.resolve_permission(
BashArgs(command="findstr /s pattern *.txt")
)
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_ver_allowlisted(self):
bash_tool = self._make_bash(allowlist=["ver"])
result = bash_tool.resolve_permission(BashArgs(command="ver"))
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_where_allowlisted(self):
bash_tool = self._make_bash(allowlist=["where"])
result = bash_tool.resolve_permission(BashArgs(command="where python"))
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_cmd_k_denylisted(self):
bash_tool = self._make_bash(denylist=["cmd /k"])
result = bash_tool.resolve_permission(BashArgs(command="cmd /k something"))
assert result is ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_powershell_noexit_denylisted(self):
bash_tool = self._make_bash(denylist=["powershell -NoExit"])
result = bash_tool.resolve_permission(BashArgs(command="powershell -NoExit"))
assert result is ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_notepad_denylisted(self):
bash_tool = self._make_bash(denylist=["notepad"])
result = bash_tool.resolve_permission(BashArgs(command="notepad file.txt"))
assert result is ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_cmd_standalone_denylisted(self):
bash_tool = self._make_bash(denylist_standalone=["cmd"])
result = bash_tool.resolve_permission(BashArgs(command="cmd"))
assert result is ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_powershell_standalone_denylisted(self):
bash_tool = self._make_bash(denylist_standalone=["powershell"])
result = bash_tool.resolve_permission(BashArgs(command="powershell"))
assert result is ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_powershell_cmdlet_asks(self):
bash_tool = self._make_bash(allowlist=["dir", "echo"])
result = bash_tool.resolve_permission(BashArgs(command="Get-ChildItem -Path ."))
assert result is None
assert isinstance(result, PermissionContext)
assert result.permission == ToolPermission.ASK
def test_mixed_allowed_and_unknown_asks(self):
bash_tool = self._make_bash(allowlist=["git status"])
result = bash_tool.resolve_permission(
BashArgs(command="git status && npm install")
)
assert result is None
assert isinstance(result, PermissionContext)
assert result.permission == ToolPermission.ASK
def test_chained_windows_commands_all_allowed(self):
bash_tool = self._make_bash(allowlist=["dir", "echo"])
result = bash_tool.resolve_permission(BashArgs(command="dir /s && echo done"))
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_chained_commands_one_denied(self):
bash_tool = self._make_bash(allowlist=["dir"], denylist=["rm"])
result = bash_tool.resolve_permission(BashArgs(command="dir /s && rm -rf /"))
assert result is ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_piped_windows_commands(self):
bash_tool = self._make_bash(allowlist=["findstr", "type"])
result = bash_tool.resolve_permission(
BashArgs(command="type file.txt | findstr pattern")
)
assert result is ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS

View file

@ -0,0 +1,750 @@
from __future__ import annotations
import os
import pytest
from vibe.core.tools.base import BaseToolState, ToolPermission
from vibe.core.tools.builtins.bash import (
Bash,
BashArgs,
BashToolConfig,
_collect_outside_dirs,
)
from vibe.core.tools.builtins.grep import Grep, GrepArgs, GrepToolConfig
from vibe.core.tools.builtins.read_file import (
ReadFile,
ReadFileArgs,
ReadFileState,
ReadFileToolConfig,
)
from vibe.core.tools.builtins.search_replace import (
SearchReplace,
SearchReplaceArgs,
SearchReplaceConfig,
)
from vibe.core.tools.builtins.webfetch import WebFetch, WebFetchArgs, WebFetchConfig
from vibe.core.tools.builtins.write_file import (
WriteFile,
WriteFileArgs,
WriteFileConfig,
)
from vibe.core.tools.permissions import (
ApprovedRule,
PermissionContext,
PermissionScope,
RequiredPermission,
)
from vibe.core.tools.utils import wildcard_match
class TestBashGranularPermissions:
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
self.workdir = tmp_path
def _bash(self, **kwargs):
config = BashToolConfig(**kwargs)
return Bash(config=config, state=BaseToolState())
def test_allowlisted_command_always(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="git status"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_denylisted_command_never(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="vim file.txt"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_standalone_denylisted_never(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="python"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_standalone_denylisted_with_args_not_denied(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="python script.py"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
def test_unknown_command_returns_permission_context(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="npm install"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
assert len(result.required_permissions) == 1
rp = result.required_permissions[0]
assert rp.scope is PermissionScope.COMMAND_PATTERN
assert rp.session_pattern == "npm install *"
def test_arity_based_prefix(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="docker compose up -d"))
assert isinstance(result, PermissionContext)
rp = result.required_permissions[0]
assert rp.session_pattern == "docker compose up *"
def test_multiple_commands_dedup(self):
bash = self._bash()
result = bash.resolve_permission(
BashArgs(command="npm install foo && npm install bar")
)
assert isinstance(result, PermissionContext)
command_labels = [
rp.label
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
assert command_labels == ["npm install *"]
def test_cd_excluded_from_command_patterns(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="cd /tmp"))
assert isinstance(result, PermissionContext)
assert all(
rp.scope is not PermissionScope.COMMAND_PATTERN
for rp in result.required_permissions
)
def test_outside_directory_detection(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="mkdir /tmp/test"))
assert isinstance(result, PermissionContext)
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside) >= 1
def test_outside_directory_has_glob_pattern(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="mkdir /tmp/test"))
assert isinstance(result, PermissionContext)
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert any("/tmp" in rp.session_pattern for rp in outside)
def test_in_workdir_no_outside_directory(self):
bash = self._bash()
(self.workdir / "subdir").mkdir()
result = bash.resolve_permission(BashArgs(command="mkdir subdir/child"))
assert isinstance(result, PermissionContext)
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside) == 0
def test_rm_uses_arity_based_pattern(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="rm -rf /tmp/something"))
assert isinstance(result, PermissionContext)
cmd_perms = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
assert len(cmd_perms) == 1
assert cmd_perms[0].session_pattern == "rm *"
def test_sensitive_sudo_exact_pattern(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="sudo apt install foo"))
assert isinstance(result, PermissionContext)
cmd_perms = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
assert cmd_perms[0].session_pattern == "sudo apt install foo"
def test_rmdir_uses_arity_based_pattern(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="rmdir foo"))
assert isinstance(result, PermissionContext)
cmd_perms = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
assert cmd_perms[0].session_pattern == "rmdir *"
def test_sensitive_bypasses_allowlist(self):
bash = self._bash(allowlist=["sudo"])
result = bash.resolve_permission(BashArgs(command="sudo ls"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
def test_allowlisted_outside_dir_still_asks(self):
bash = self._bash()
# cat is allowlisted but /etc/passwd is outside workdir
result = bash.resolve_permission(BashArgs(command="cat /etc/passwd"))
assert isinstance(result, PermissionContext)
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside) == 1
def test_allowlisted_relative_traversal_outside_dir_still_asks(self):
bash = self._bash()
(self.workdir / "src").mkdir()
result = bash.resolve_permission(
BashArgs(command="cat src/../../../etc/passwd")
)
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside) >= 1
def test_allowlisted_in_workdir_subdir_auto_approves(self):
bash = self._bash()
(self.workdir / "foo").mkdir()
(self.workdir / "foo" / "bar.txt").touch()
result = bash.resolve_permission(BashArgs(command="cat foo/bar.txt"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_allowlisted_in_workdir_auto_approves(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="cat README.md"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_mixed_allowlisted_and_not(self):
bash = self._bash()
result = bash.resolve_permission(
BashArgs(command="echo hello && npm install foo")
)
assert isinstance(result, PermissionContext)
cmd_perms = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
assert len(cmd_perms) == 1
assert cmd_perms[0].session_pattern == "npm install *"
def test_empty_command_returns_none(self):
bash = self._bash()
assert bash.resolve_permission(BashArgs(command="")) is None
def test_chmod_plus_skipped_as_flag(self):
bash = self._bash()
result = bash.resolve_permission(BashArgs(command="chmod +x /tmp/script.sh"))
assert isinstance(result, PermissionContext)
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside) >= 1
class TestReadFileGranularPermissions:
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
self.workdir = tmp_path
def _read_file(self, **kwargs):
config = ReadFileToolConfig(**kwargs)
return ReadFile(config=config, state=ReadFileState())
def test_in_workdir_normal_file_returns_none(self):
(self.workdir / "test.py").touch()
tool = self._read_file()
assert tool.resolve_permission(ReadFileArgs(path="test.py")) is None
def test_outside_workdir_returns_permission_context(self):
tool = self._read_file()
result = tool.resolve_permission(ReadFileArgs(path="/tmp/file.txt"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
outside = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside) == 1
def test_sensitive_env_file_returns_permission_context(self):
(self.workdir / ".env").touch()
tool = self._read_file()
result = tool.resolve_permission(ReadFileArgs(path=".env"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
sensitive = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.FILE_PATTERN
]
assert len(sensitive) == 1
assert sensitive[0].label.startswith("accessing sensitive files")
def test_sensitive_env_local_file(self):
(self.workdir / ".env.local").touch()
tool = self._read_file()
result = tool.resolve_permission(ReadFileArgs(path=".env.local"))
assert isinstance(result, PermissionContext)
sensitive = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.FILE_PATTERN
]
assert len(sensitive) == 1
def test_sensitive_outside_both_permissions(self):
tool = self._read_file()
result = tool.resolve_permission(ReadFileArgs(path="/tmp/.env"))
assert isinstance(result, PermissionContext)
scopes = {rp.scope for rp in result.required_permissions}
assert PermissionScope.FILE_PATTERN in scopes
assert PermissionScope.OUTSIDE_DIRECTORY in scopes
def test_denylisted_returns_never(self):
tool = self._read_file(denylist=["*/secret*"])
result = tool.resolve_permission(ReadFileArgs(path="secret.key"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_allowlisted_returns_always(self):
tool = self._read_file(allowlist=["*/README*"])
result = tool.resolve_permission(
ReadFileArgs(path=str(self.workdir / "README.md"))
)
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_custom_sensitive_patterns(self):
(self.workdir / "credentials.json").touch()
tool = self._read_file(sensitive_patterns=["*/credentials*"])
result = tool.resolve_permission(ReadFileArgs(path="credentials.json"))
assert isinstance(result, PermissionContext)
class TestWriteFileGranularPermissions:
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
self.workdir = tmp_path
def _write_file(self):
config = WriteFileConfig()
return WriteFile(config=config, state=BaseToolState())
def test_in_workdir_returns_none(self):
tool = self._write_file()
assert (
tool.resolve_permission(WriteFileArgs(path="test.py", content="x")) is None
)
def test_outside_workdir_returns_permission_context(self):
tool = self._write_file()
result = tool.resolve_permission(
WriteFileArgs(path="/tmp/file.txt", content="x")
)
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
def test_sensitive_env_file_asks(self):
(self.workdir / ".env").touch()
tool = self._write_file()
result = tool.resolve_permission(
WriteFileArgs(path=".env", content="x", overwrite=True)
)
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
class TestSearchReplaceGranularPermissions:
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
def test_outside_workdir_returns_permission_context(self):
config = SearchReplaceConfig()
tool = SearchReplace(config=config, state=BaseToolState())
result = tool.resolve_permission(
SearchReplaceArgs(file_path="/tmp/file.py", content="x")
)
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ASK
class TestGrepGranularPermissions:
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
self.workdir = tmp_path
def _grep(self):
config = GrepToolConfig()
return Grep(config=config, state=BaseToolState())
def test_in_workdir_normal_path_returns_none(self):
tool = self._grep()
assert tool.resolve_permission(GrepArgs(pattern="foo", path=".")) is None
def test_outside_workdir_returns_permission_context(self):
tool = self._grep()
result = tool.resolve_permission(GrepArgs(pattern="foo", path="/tmp"))
assert isinstance(result, PermissionContext)
def test_sensitive_env_directory(self):
(self.workdir / ".env").touch()
tool = self._grep()
result = tool.resolve_permission(GrepArgs(pattern="foo", path=".env"))
assert isinstance(result, PermissionContext)
sensitive = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.FILE_PATTERN
]
assert len(sensitive) == 1
class TestApprovalFlowSimulation:
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
def _is_covered(
self, tool_name: str, rp: RequiredPermission, rules: list[ApprovedRule]
) -> bool:
return any(
rule.tool_name == tool_name
and rule.scope == rp.scope
and wildcard_match(rp.invocation_pattern, rule.session_pattern)
for rule in rules
)
def test_mkdir_approved_covers_subsequent_mkdir(self):
rules = [
ApprovedRule(
tool_name="bash",
scope=PermissionScope.COMMAND_PATTERN,
session_pattern="mkdir *",
)
]
bash = Bash(config=BashToolConfig(), state=BaseToolState())
result = bash.resolve_permission(BashArgs(command="mkdir another_dir"))
assert isinstance(result, PermissionContext)
uncovered = [
rp
for rp in result.required_permissions
if not self._is_covered("bash", rp, rules)
]
assert not any(rp.scope is PermissionScope.COMMAND_PATTERN for rp in uncovered)
def test_mkdir_approved_does_not_cover_npm(self):
rules = [
ApprovedRule(
tool_name="bash",
scope=PermissionScope.COMMAND_PATTERN,
session_pattern="mkdir *",
)
]
bash = Bash(config=BashToolConfig(), state=BaseToolState())
result = bash.resolve_permission(BashArgs(command="npm install"))
assert isinstance(result, PermissionContext)
uncovered = [
rp
for rp in result.required_permissions
if not self._is_covered("bash", rp, rules)
]
assert len(uncovered) == 1
assert uncovered[0].session_pattern == "npm install *"
def test_outside_dir_approved_covers_subsequent(self):
bash = Bash(config=BashToolConfig(), state=BaseToolState())
result = bash.resolve_permission(BashArgs(command="mkdir /tmp/newdir"))
assert isinstance(result, PermissionContext)
outside_rps = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.OUTSIDE_DIRECTORY
]
assert len(outside_rps) == 1
# Resolved pattern may differ per OS (e.g. /private/tmp/* on macOS)
rules = [
ApprovedRule(
tool_name="bash",
scope=PermissionScope.OUTSIDE_DIRECTORY,
session_pattern=outside_rps[0].session_pattern,
),
ApprovedRule(
tool_name="bash",
scope=PermissionScope.COMMAND_PATTERN,
session_pattern="mkdir *",
),
]
uncovered = [
rp
for rp in result.required_permissions
if not self._is_covered("bash", rp, rules)
]
assert len(uncovered) == 0
def test_rm_approved_covers_subsequent_rm(self):
rules = [
ApprovedRule(
tool_name="bash",
scope=PermissionScope.COMMAND_PATTERN,
session_pattern="rm *",
)
]
bash = Bash(config=BashToolConfig(), state=BaseToolState())
result = bash.resolve_permission(BashArgs(command="rm -rf /tmp/something"))
assert isinstance(result, PermissionContext)
cmd_perms = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
assert cmd_perms[0].session_pattern == "rm *"
uncovered = [rp for rp in cmd_perms if not self._is_covered("bash", rp, rules)]
assert len(uncovered) == 0
def test_sudo_exact_approval_doesnt_cover_different_invocation(self):
rules = [
ApprovedRule(
tool_name="bash",
scope=PermissionScope.COMMAND_PATTERN,
session_pattern="sudo apt install foo",
)
]
bash = Bash(config=BashToolConfig(), state=BaseToolState())
result = bash.resolve_permission(BashArgs(command="sudo apt install bar"))
assert isinstance(result, PermissionContext)
cmd_perms = [
rp
for rp in result.required_permissions
if rp.scope is PermissionScope.COMMAND_PATTERN
]
uncovered = [rp for rp in cmd_perms if not self._is_covered("bash", rp, rules)]
assert len(uncovered) == 1
def test_read_file_sensitive_approved_covers_subsequent(self):
rules = [
ApprovedRule(
tool_name="read_file",
scope=PermissionScope.FILE_PATTERN,
session_pattern="*",
)
]
rp = RequiredPermission(
scope=PermissionScope.FILE_PATTERN,
invocation_pattern=".env.production",
session_pattern="*",
label="reading sensitive files (read_file)",
)
assert self._is_covered("read_file", rp, rules)
def test_different_tool_rule_doesnt_cover(self):
rules = [
ApprovedRule(
tool_name="bash",
scope=PermissionScope.COMMAND_PATTERN,
session_pattern="mkdir *",
)
]
rp = RequiredPermission(
scope=PermissionScope.COMMAND_PATTERN,
invocation_pattern="mkdir foo",
session_pattern="mkdir *",
label="mkdir *",
)
assert not self._is_covered("grep", rp, rules)
class TestWebFetchPermissions:
def _make_webfetch(self) -> WebFetch:
return WebFetch(config=WebFetchConfig(), state=BaseToolState())
def test_returns_url_pattern_with_domain(self):
wf = self._make_webfetch()
result = wf.resolve_permission(
WebFetchArgs(url="https://docs.python.org/3/library")
)
assert isinstance(result, PermissionContext)
assert len(result.required_permissions) == 1
rp = result.required_permissions[0]
assert rp.scope is PermissionScope.URL_PATTERN
assert rp.invocation_pattern == "docs.python.org"
assert rp.session_pattern == "docs.python.org"
assert "docs.python.org" in rp.label
def test_http_url(self):
wf = self._make_webfetch()
result = wf.resolve_permission(WebFetchArgs(url="http://example.com/page"))
assert isinstance(result, PermissionContext)
rp = result.required_permissions[0]
assert rp.invocation_pattern == "example.com"
def test_url_without_scheme(self):
wf = self._make_webfetch()
result = wf.resolve_permission(WebFetchArgs(url="github.com/anthropics"))
assert isinstance(result, PermissionContext)
rp = result.required_permissions[0]
assert rp.invocation_pattern == "github.com"
def test_url_with_port(self):
wf = self._make_webfetch()
result = wf.resolve_permission(WebFetchArgs(url="http://localhost:8080/api"))
assert isinstance(result, PermissionContext)
rp = result.required_permissions[0]
assert rp.invocation_pattern == "localhost:8080"
def test_url_without_scheme_with_port(self):
wf = self._make_webfetch()
result = wf.resolve_permission(WebFetchArgs(url="example.com:3000/path"))
assert isinstance(result, PermissionContext)
rp = result.required_permissions[0]
assert rp.invocation_pattern == "example.com:3000"
def test_different_domains_not_covered(self):
rules = [
ApprovedRule(
tool_name="web_fetch",
scope=PermissionScope.URL_PATTERN,
session_pattern="docs.python.org",
)
]
rp = RequiredPermission(
scope=PermissionScope.URL_PATTERN,
invocation_pattern="evil.com",
session_pattern="evil.com",
label="fetching from evil.com",
)
covered = any(
rule.tool_name == "web_fetch"
and rule.scope == rp.scope
and wildcard_match(rp.invocation_pattern, rule.session_pattern)
for rule in rules
)
assert not covered
def test_same_domain_covered(self):
rules = [
ApprovedRule(
tool_name="web_fetch",
scope=PermissionScope.URL_PATTERN,
session_pattern="docs.python.org",
)
]
rp = RequiredPermission(
scope=PermissionScope.URL_PATTERN,
invocation_pattern="docs.python.org",
session_pattern="docs.python.org",
label="fetching from docs.python.org",
)
covered = any(
rule.tool_name == "web_fetch"
and rule.scope == rp.scope
and wildcard_match(rp.invocation_pattern, rule.session_pattern)
for rule in rules
)
assert covered
def test_double_slash_url(self):
wf = self._make_webfetch()
result = wf.resolve_permission(WebFetchArgs(url="//cdn.example.com/lib.js"))
assert isinstance(result, PermissionContext)
rp = result.required_permissions[0]
assert rp.invocation_pattern == "cdn.example.com"
def test_config_permission_always_honored(self):
wf = WebFetch(
config=WebFetchConfig(permission=ToolPermission.ALWAYS),
state=BaseToolState(),
)
result = wf.resolve_permission(WebFetchArgs(url="https://example.com"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_config_permission_never_honored(self):
wf = WebFetch(
config=WebFetchConfig(permission=ToolPermission.NEVER),
state=BaseToolState(),
)
result = wf.resolve_permission(WebFetchArgs(url="https://example.com"))
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_config_permission_ask_falls_through_to_domain(self):
wf = WebFetch(
config=WebFetchConfig(permission=ToolPermission.ASK), state=BaseToolState()
)
result = wf.resolve_permission(WebFetchArgs(url="https://example.com"))
assert isinstance(result, PermissionContext)
assert result.required_permissions[0].invocation_pattern == "example.com"
class TestCollectOutsideDirs:
"""Tests for _collect_outside_dirs helper."""
@pytest.fixture(autouse=True)
def _setup(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
self.workdir = tmp_path
def test_relative_path_resolving_outside_workdir(self):
dirs = _collect_outside_dirs(["cat ../../etc/passwd"])
# The relative path resolves outside workdir, should collect parent dir
assert len(dirs) >= 1
def test_multiple_targets_in_one_command(self):
dirs = _collect_outside_dirs(["cp /tmp/a /var/b"])
assert len(dirs) == 2
def test_chmod_skips_plus_x_token(self):
dirs = _collect_outside_dirs(["chmod +x /tmp/script.sh"])
# +x should be skipped, only /tmp/script.sh should be considered
assert len(dirs) >= 1
# Verify no dir was created from the "+x" token
for d in dirs:
assert "+x" not in d
def test_empty_command_list(self):
assert _collect_outside_dirs([]) == set()
def test_home_relative_path(self):
home = os.path.expanduser("~")
dirs = _collect_outside_dirs(["cat ~/some_file"])
# ~/some_file resolves to home directory, which is likely outside workdir
if home != str(self.workdir):
assert len(dirs) >= 1
def test_in_workdir_path_not_collected(self):
(self.workdir / "local_file").touch()
dirs = _collect_outside_dirs(["cat ./local_file"])
assert len(dirs) == 0
def test_traversal_path_without_dot_prefix(self):
"""Paths like src/../../../etc/passwd don't start with . but contain /."""
(self.workdir / "src").mkdir()
dirs = _collect_outside_dirs(["cat src/../../../etc/passwd"])
assert len(dirs) >= 1
def test_in_workdir_subdir_path_not_collected(self):
"""foo/bar inside workdir should not be flagged."""
(self.workdir / "foo").mkdir()
(self.workdir / "foo" / "bar").touch()
dirs = _collect_outside_dirs(["cat foo/bar"])
assert len(dirs) == 0

View file

@ -46,7 +46,10 @@ class TestInvokeContext:
def test_approval_callback_can_be_set(self) -> None:
async def dummy_callback(
_tool_name: str, _args: BaseModel, _tool_call_id: str
_tool_name: str,
_args: BaseModel,
_tool_call_id: str,
_rp: list | None = None,
) -> tuple[ApprovalResponse, str | None]:
return ApprovalResponse.YES, None
@ -76,7 +79,10 @@ class TestToolInvokeWithContext:
@pytest.mark.asyncio
async def test_invoke_with_approval_callback(self, simple_tool: SimpleTool) -> None:
async def dummy_callback(
_tool_name: str, _args: BaseModel, _tool_call_id: str
_tool_name: str,
_args: BaseModel,
_tool_call_id: str,
_rp: list | None = None,
) -> tuple[ApprovalResponse, str | None]:
return ApprovalResponse.YES, None

View file

@ -36,7 +36,7 @@ def test_merges_user_overrides_with_defaults():
vibe_config = build_test_vibe_config(
system_prompt_id="tests",
include_project_context=False,
tools={"bash": BaseToolConfig(permission=ToolPermission.ALWAYS)},
tools={"bash": {"permission": "always"}},
)
manager = ToolManager(lambda: vibe_config)
@ -53,9 +53,9 @@ def test_preserves_tool_specific_fields_from_overrides():
vibe_config = build_test_vibe_config(
system_prompt_id="tests",
include_project_context=False,
tools={"bash": BaseToolConfig(permission=ToolPermission.ASK)},
tools={"bash": {"permission": "ask"}},
)
vibe_config.tools["bash"].__pydantic_extra__ = {"default_timeout": 600}
vibe_config.tools["bash"]["default_timeout"] = 600
manager = ToolManager(lambda: vibe_config)
config = manager.get_tool_config("bash")
@ -71,6 +71,23 @@ def test_falls_back_to_base_config_for_unknown_tool(tool_manager):
assert config.permission == ToolPermission.ASK
def test_partial_override_preserves_tool_defaults():
vibe_config = build_test_vibe_config(
system_prompt_id="tests",
include_project_context=False,
tools={"read_file": {"max_read_bytes": 32000}},
)
manager = ToolManager(lambda: vibe_config)
config = manager.get_tool_config("read_file")
assert (
config.permission == ToolPermission.ALWAYS
) # ReadFileToolConfig default, not BaseToolConfig.ASK
assert config.sensitive_patterns == ["**/.env", "**/.env.*"] # type: ignore[attr-defined]
assert config.max_read_bytes == 32000 # type: ignore[attr-defined]
class TestToolManagerFiltering:
def test_enabled_tools_filters_to_only_enabled(self):
vibe_config = build_test_vibe_config(

217
tests/tools/test_skill.py Normal file
View file

@ -0,0 +1,217 @@
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock
import pytest
from tests.mock.utils import collect_result
from vibe.core.skills.manager import SkillManager
from vibe.core.skills.models import SkillInfo
from vibe.core.tools.base import BaseToolState, InvokeContext, ToolError
from vibe.core.tools.builtins.skill import (
Skill,
SkillArgs,
SkillResult,
SkillToolConfig,
)
from vibe.core.tools.permissions import PermissionScope
def _make_skill_dir(
tmp_path: Path,
name: str = "my-skill",
description: str = "A test skill",
body: str = "## Instructions\n\nDo the thing.",
extra_files: list[str] | None = None,
) -> SkillInfo:
skill_dir = tmp_path / name
skill_dir.mkdir(parents=True, exist_ok=True)
content = f"---\nname: {name}\ndescription: {description}\n---\n\n{body}"
(skill_dir / "SKILL.md").write_text(content, encoding="utf-8")
for f in extra_files or []:
file_path = skill_dir / f
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_text(f"content of {f}", encoding="utf-8")
return SkillInfo(
name=name, description=description, skill_path=skill_dir / "SKILL.md"
)
def _make_skill_manager(skills: dict[str, SkillInfo]) -> SkillManager:
manager = MagicMock(spec=SkillManager)
manager.available_skills = skills
manager.get_skill.side_effect = lambda n: skills.get(n)
return manager
def _make_ctx(skill_manager: SkillManager | None = None) -> InvokeContext:
return InvokeContext(tool_call_id="test-call", skill_manager=skill_manager)
@pytest.fixture
def skill_tool() -> Skill:
return Skill(config=SkillToolConfig(), state=BaseToolState())
class TestSkillRun:
@pytest.mark.asyncio
async def test_loads_skill_content(self, tmp_path: Path, skill_tool: Skill) -> None:
info = _make_skill_dir(tmp_path, body="Follow these steps:\n1. Do A\n2. Do B")
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
assert isinstance(result, SkillResult)
assert result.name == "my-skill"
assert "Follow these steps:" in result.content
assert "1. Do A" in result.content
assert '<skill_content name="my-skill">' in result.content
assert "# Skill: my-skill" in result.content
assert "</skill_content>" in result.content
@pytest.mark.asyncio
async def test_lists_bundled_files(self, tmp_path: Path, skill_tool: Skill) -> None:
info = _make_skill_dir(
tmp_path, extra_files=["scripts/run.sh", "references/guide.md"]
)
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
assert "<skill_files>" in result.content
assert "<file>scripts/run.sh</file>" in result.content
assert "<file>references/guide.md</file>" in result.content
assert f"<file>{info.skill_dir / 'scripts/run.sh'}</file>" not in result.content
@pytest.mark.asyncio
async def test_excludes_skill_md_from_file_list(
self, tmp_path: Path, skill_tool: Skill
) -> None:
info = _make_skill_dir(tmp_path, extra_files=["helper.py"])
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
assert "SKILL.md" not in result.content.split("<skill_files>")[1]
assert "helper.py" in result.content
@pytest.mark.asyncio
async def test_caps_file_list_at_ten(
self, tmp_path: Path, skill_tool: Skill
) -> None:
files = [f"file{i:02d}.txt" for i in range(15)]
info = _make_skill_dir(tmp_path, extra_files=files)
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
file_section = result.content.split("<skill_files>")[1].split("</skill_files>")[
0
]
assert file_section.count("<file>") == 10
@pytest.mark.asyncio
async def test_empty_skill_directory(
self, tmp_path: Path, skill_tool: Skill
) -> None:
info = _make_skill_dir(tmp_path)
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
assert "<skill_files>\n\n</skill_files>" in result.content
@pytest.mark.asyncio
async def test_returns_skill_dir(self, tmp_path: Path, skill_tool: Skill) -> None:
info = _make_skill_dir(tmp_path)
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
assert result.skill_dir == str(info.skill_dir)
@pytest.mark.asyncio
async def test_includes_base_directory(
self, tmp_path: Path, skill_tool: Skill
) -> None:
info = _make_skill_dir(tmp_path)
manager = _make_skill_manager({"my-skill": info})
ctx = _make_ctx(manager)
result = await collect_result(skill_tool.run(SkillArgs(name="my-skill"), ctx))
assert f"Base directory for this skill: {info.skill_dir}" in result.content
class TestSkillErrors:
@pytest.mark.asyncio
async def test_no_context(self, skill_tool: Skill) -> None:
with pytest.raises(ToolError, match="Skill manager not available"):
await collect_result(skill_tool.run(SkillArgs(name="test"), ctx=None))
@pytest.mark.asyncio
async def test_no_skill_manager(self, skill_tool: Skill) -> None:
ctx = _make_ctx(skill_manager=None)
with pytest.raises(ToolError, match="Skill manager not available"):
await collect_result(skill_tool.run(SkillArgs(name="test"), ctx=ctx))
@pytest.mark.asyncio
async def test_skill_not_found(self, skill_tool: Skill) -> None:
manager = _make_skill_manager({"alpha": MagicMock(), "beta": MagicMock()})
ctx = _make_ctx(manager)
with pytest.raises(ToolError, match='Skill "missing" not found'):
await collect_result(skill_tool.run(SkillArgs(name="missing"), ctx=ctx))
@pytest.mark.asyncio
async def test_skill_not_found_lists_available(self, skill_tool: Skill) -> None:
manager = _make_skill_manager({"alpha": MagicMock(), "beta": MagicMock()})
ctx = _make_ctx(manager)
with pytest.raises(ToolError, match="alpha, beta"):
await collect_result(skill_tool.run(SkillArgs(name="missing"), ctx=ctx))
@pytest.mark.asyncio
async def test_unreadable_skill_file(
self, tmp_path: Path, skill_tool: Skill
) -> None:
info = SkillInfo(
name="broken",
description="Broken skill",
skill_path=tmp_path / "nonexistent" / "SKILL.md",
)
manager = _make_skill_manager({"broken": info})
ctx = _make_ctx(manager)
with pytest.raises(ToolError, match="Cannot load skill file"):
await collect_result(skill_tool.run(SkillArgs(name="broken"), ctx=ctx))
class TestSkillPermission:
def test_resolve_permission_returns_file_pattern(self, skill_tool: Skill) -> None:
perm = skill_tool.resolve_permission(SkillArgs(name="my-skill"))
assert perm is not None
assert len(perm.required_permissions) == 1
assert perm.required_permissions[0].scope == PermissionScope.FILE_PATTERN
assert perm.required_permissions[0].invocation_pattern == "my-skill"
assert perm.required_permissions[0].session_pattern == "my-skill"
class TestSkillMeta:
def test_tool_name(self) -> None:
assert Skill.get_name() == "skill"
def test_description_is_set(self) -> None:
assert "skill" in Skill.description.lower()
assert len(Skill.description) > 20

View file

@ -10,6 +10,7 @@ from vibe.core.agents.manager import AgentManager
from vibe.core.agents.models import BUILTIN_AGENTS, AgentType
from vibe.core.tools.base import BaseToolState, InvokeContext, ToolError, ToolPermission
from vibe.core.tools.builtins.task import Task, TaskArgs, TaskResult, TaskToolConfig
from vibe.core.tools.permissions import PermissionContext
from vibe.core.types import AssistantEvent, LLMMessage, Role
@ -80,7 +81,8 @@ class TestTaskToolResolvePermission:
def test_explore_allowed_by_default(self, task_tool: Task) -> None:
args = TaskArgs(task="do something", agent="explore")
result = task_tool.resolve_permission(args)
assert result == ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_unknown_agent_returns_none(self, task_tool: Task) -> None:
args = TaskArgs(task="do something", agent="custom_agent")
@ -92,21 +94,24 @@ class TestTaskToolResolvePermission:
tool = Task(config=config, state=BaseToolState())
args = TaskArgs(task="do something", agent="explore")
result = tool.resolve_permission(args)
assert result == ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_glob_pattern_in_allowlist(self) -> None:
config = TaskToolConfig(allowlist=["exp*"])
tool = Task(config=config, state=BaseToolState())
args = TaskArgs(task="do something", agent="explore")
result = tool.resolve_permission(args)
assert result == ToolPermission.ALWAYS
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.ALWAYS
def test_glob_pattern_in_denylist(self) -> None:
config = TaskToolConfig(denylist=["danger*"])
tool = Task(config=config, state=BaseToolState())
args = TaskArgs(task="do something", agent="dangerous_agent")
result = tool.resolve_permission(args)
assert result == ToolPermission.NEVER
assert isinstance(result, PermissionContext)
assert result.permission is ToolPermission.NEVER
def test_empty_lists_returns_none(self) -> None:
config = TaskToolConfig(allowlist=[], denylist=[])

View file

@ -14,9 +14,10 @@ from mistralai.client.models import (
import pytest
from tests.mock.utils import collect_result
from vibe.core.config import Backend, ProviderConfig
from vibe.core.config import ProviderConfig
from vibe.core.tools.base import BaseToolState, InvokeContext, ToolError
from vibe.core.tools.builtins.websearch import WebSearch, WebSearchArgs, WebSearchConfig
from vibe.core.types import Backend
def _make_response(

View file

@ -0,0 +1,66 @@
from __future__ import annotations
from vibe.core.tools.utils import wildcard_match
class TestWildcardMatch:
def test_exact_match(self):
assert wildcard_match("hello", "hello")
def test_exact_no_match(self):
assert not wildcard_match("hello", "world")
def test_star_matches_any(self):
assert wildcard_match("hello world", "hello *")
def test_star_matches_empty_trailing(self):
assert wildcard_match("mkdir", "mkdir *")
def test_star_matches_with_args(self):
assert wildcard_match("mkdir hello", "mkdir *")
def test_star_matches_long_trailing(self):
assert wildcard_match("git commit -m hello world", "git commit *")
def test_star_in_middle(self):
assert wildcard_match("fooXbar", "foo*bar")
def test_question_mark_single_char(self):
assert wildcard_match("cat", "c?t")
def test_question_mark_no_match(self):
assert not wildcard_match("ct", "c?t")
def test_glob_path_pattern(self):
assert wildcard_match("/tmp/dir/file.txt", "/tmp/dir/*")
def test_glob_nested_path(self):
assert wildcard_match("/tmp/dir/sub/file.txt", "/tmp/dir/*")
def test_glob_no_match(self):
assert not wildcard_match("/home/user/file.txt", "/tmp/*")
def test_special_regex_chars_in_text(self):
assert wildcard_match("echo (hello)", "echo *")
def test_special_regex_chars_in_pattern(self):
assert wildcard_match(".env", ".env")
assert not wildcard_match("xenv", ".env")
def test_fnmatch_character_class(self):
assert wildcard_match("vache", "[bcghlmstv]ache")
def test_empty_pattern_empty_text(self):
assert wildcard_match("", "")
def test_star_only(self):
assert wildcard_match("anything goes here", "*")
def test_trailing_space_star_is_optional(self):
assert wildcard_match("ls", "ls *")
assert wildcard_match("ls -la", "ls *")
assert wildcard_match("ls -la /tmp", "ls *")
def test_non_trailing_star_is_greedy(self):
assert wildcard_match("abc123def", "abc*def")
assert not wildcard_match("abc123de", "abc*def")