v2.7.0 (#534)
Co-authored-by: Quentin Torroba <quentin.torroba@mistral.ai> Co-authored-by: Clément Drouin <clement.drouin@mistral.ai> Co-authored-by: Clément Sirieix <clement.sirieix@mistral.ai> Co-authored-by: Mistral Vibe <vibe@mistral.ai>
This commit is contained in:
Mathias Gesbert
GitHub
parent
5ea69b547b
commit
6a50d1d521
49 changed files with 3726 additions and 99 deletions
|
|
@ -13,9 +13,11 @@ the tests will be. Always prefer real API data over manually constructed example
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
from typing import ClassVar, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
from mistralai.client.errors import SDKError
|
||||
from mistralai.client.models import AssistantMessage
|
||||
from mistralai.client.utils.retries import BackoffStrategy, RetryConfig
|
||||
import pytest
|
||||
|
|
@ -38,7 +40,7 @@ from vibe.core.config import ModelConfig, ProviderConfig
|
|||
from vibe.core.llm.backend.factory import BACKEND_FACTORY
|
||||
from vibe.core.llm.backend.generic import GenericBackend
|
||||
from vibe.core.llm.backend.mistral import MistralBackend, MistralMapper
|
||||
from vibe.core.llm.exceptions import BackendError
|
||||
from vibe.core.llm.exceptions import BackendError, BackendErrorBuilder
|
||||
from vibe.core.llm.types import BackendLike
|
||||
from vibe.core.types import Backend, FunctionCall, LLMChunk, LLMMessage, Role, ToolCall
|
||||
from vibe.core.utils import get_user_agent
|
||||
|
|
@ -526,3 +528,237 @@ class TestMistralMapperPrepareMessage:
|
|||
msg = LLMMessage(role=Role.assistant, content="Hello!")
|
||||
result = mapper.prepare_message(msg)
|
||||
assert result.content == "Hello!"
|
||||
|
||||
def test_strip_reasoning_removes_reasoning_from_assistant(
|
||||
self, mapper: MistralMapper
|
||||
) -> None:
|
||||
msg = LLMMessage(
|
||||
role=Role.assistant,
|
||||
content="Answer",
|
||||
reasoning_content="thinking...",
|
||||
reasoning_signature="sig",
|
||||
)
|
||||
stripped = mapper.strip_reasoning(msg)
|
||||
assert stripped.content == "Answer"
|
||||
assert stripped.reasoning_content is None
|
||||
assert stripped.reasoning_signature is None
|
||||
|
||||
def test_strip_reasoning_leaves_non_assistant_unchanged(
|
||||
self, mapper: MistralMapper
|
||||
) -> None:
|
||||
msg = LLMMessage(role=Role.user, content="hello")
|
||||
assert mapper.strip_reasoning(msg) is msg
|
||||
|
||||
|
||||
class TestMistralBackendReasoningEffort:
|
||||
"""Tests that MistralBackend correctly passes reasoning_effort to the SDK."""
|
||||
|
||||
@pytest.fixture
|
||||
def backend(self) -> MistralBackend:
|
||||
provider = ProviderConfig(
|
||||
name="mistral",
|
||||
api_base="https://api.mistral.ai/v1",
|
||||
api_key_env_var="API_KEY",
|
||||
)
|
||||
return MistralBackend(provider=provider)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("thinking", "expected_effort", "expected_temperature"),
|
||||
[
|
||||
("off", None, 0.2),
|
||||
("low", "none", 1.0),
|
||||
("medium", "high", 1.0),
|
||||
("high", "high", 1.0),
|
||||
],
|
||||
)
|
||||
async def test_complete_passes_reasoning_effort(
|
||||
self,
|
||||
backend: MistralBackend,
|
||||
thinking: Literal["off", "low", "medium", "high"],
|
||||
expected_effort: str | None,
|
||||
expected_temperature: float,
|
||||
) -> None:
|
||||
model = ModelConfig(
|
||||
name="mistral-small-latest",
|
||||
provider="mistral",
|
||||
alias="mistral-small",
|
||||
thinking=thinking,
|
||||
)
|
||||
messages = [LLMMessage(role=Role.user, content="hi")]
|
||||
|
||||
with patch.object(backend, "_get_client") as mock_get_client:
|
||||
mock_client = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "hello"
|
||||
mock_response.choices[0].message.tool_calls = None
|
||||
mock_response.usage.prompt_tokens = 10
|
||||
mock_response.usage.completion_tokens = 5
|
||||
mock_client.chat.complete_async = AsyncMock(return_value=mock_response)
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
await backend.complete(
|
||||
model=model,
|
||||
messages=messages,
|
||||
temperature=0.2,
|
||||
tools=None,
|
||||
max_tokens=None,
|
||||
tool_choice=None,
|
||||
extra_headers=None,
|
||||
)
|
||||
|
||||
call_kwargs = mock_client.chat.complete_async.call_args.kwargs
|
||||
assert call_kwargs["reasoning_effort"] == expected_effort
|
||||
assert call_kwargs["temperature"] == expected_temperature
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_strips_reasoning_when_thinking_off(
|
||||
self, backend: MistralBackend
|
||||
) -> None:
|
||||
model = ModelConfig(
|
||||
name="mistral-small-latest",
|
||||
provider="mistral",
|
||||
alias="mistral-small",
|
||||
thinking="off",
|
||||
)
|
||||
messages = [
|
||||
LLMMessage(role=Role.user, content="hi"),
|
||||
LLMMessage(
|
||||
role=Role.assistant, content="answer", reasoning_content="thinking..."
|
||||
),
|
||||
LLMMessage(role=Role.user, content="follow up"),
|
||||
]
|
||||
|
||||
with patch.object(backend, "_get_client") as mock_get_client:
|
||||
mock_client = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [MagicMock()]
|
||||
mock_response.choices[0].message.content = "response"
|
||||
mock_response.choices[0].message.tool_calls = None
|
||||
mock_response.usage.prompt_tokens = 10
|
||||
mock_response.usage.completion_tokens = 5
|
||||
mock_client.chat.complete_async = AsyncMock(return_value=mock_response)
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
await backend.complete(
|
||||
model=model,
|
||||
messages=messages,
|
||||
temperature=0.2,
|
||||
tools=None,
|
||||
max_tokens=None,
|
||||
tool_choice=None,
|
||||
extra_headers=None,
|
||||
)
|
||||
|
||||
call_kwargs = mock_client.chat.complete_async.call_args.kwargs
|
||||
assert call_kwargs["reasoning_effort"] is None
|
||||
# The assistant message should have reasoning stripped
|
||||
converted_msgs = call_kwargs["messages"]
|
||||
assistant_msg = converted_msgs[1]
|
||||
assert isinstance(assistant_msg, AssistantMessage)
|
||||
assert assistant_msg.content == "answer"
|
||||
|
||||
|
||||
class TestBuildHttpErrorBodyReading:
|
||||
_MESSAGES: ClassVar[list[LLMMessage]] = [LLMMessage(role=Role.user, content="hi")]
|
||||
_COMMON_KWARGS: ClassVar[dict] = dict(
|
||||
provider="test",
|
||||
endpoint="https://api.test.com",
|
||||
model="test-model",
|
||||
messages=_MESSAGES,
|
||||
temperature=0.2,
|
||||
has_tools=False,
|
||||
tool_choice=None,
|
||||
)
|
||||
|
||||
def _make_sdk_error(self, response: httpx.Response) -> SDKError:
|
||||
return SDKError("sdk error", response)
|
||||
|
||||
def _make_http_status_error(
|
||||
self, response: httpx.Response
|
||||
) -> httpx.HTTPStatusError:
|
||||
return httpx.HTTPStatusError(
|
||||
"http error", request=response.request, response=response
|
||||
)
|
||||
|
||||
def test_sdk_error_readable_body(self) -> None:
|
||||
response = httpx.Response(
|
||||
400,
|
||||
json={"message": "invalid temperature"},
|
||||
request=httpx.Request("POST", "https://api.test.com"),
|
||||
)
|
||||
err = BackendErrorBuilder.build_http_error(
|
||||
error=self._make_sdk_error(response), **self._COMMON_KWARGS
|
||||
)
|
||||
assert err.status == 400
|
||||
assert err.parsed_error == "invalid temperature"
|
||||
assert "invalid temperature" in err.body_text
|
||||
|
||||
def test_http_status_error_readable_body(self) -> None:
|
||||
response = httpx.Response(
|
||||
400,
|
||||
json={"message": "invalid temperature"},
|
||||
request=httpx.Request("POST", "https://api.test.com"),
|
||||
)
|
||||
err = BackendErrorBuilder.build_http_error(
|
||||
error=self._make_http_status_error(response), **self._COMMON_KWARGS
|
||||
)
|
||||
assert err.status == 400
|
||||
assert err.parsed_error == "invalid temperature"
|
||||
assert "invalid temperature" in err.body_text
|
||||
|
||||
def test_sdk_error_stream_response_falls_back_to_read(self) -> None:
|
||||
response = httpx.Response(
|
||||
400,
|
||||
stream=httpx.ByteStream(b'{"message": "context too long"}'),
|
||||
request=httpx.Request("POST", "https://api.test.com"),
|
||||
)
|
||||
sdk_err = SDKError(
|
||||
"sdk error", response, body='{"message": "context too long"}'
|
||||
)
|
||||
err = BackendErrorBuilder.build_http_error(error=sdk_err, **self._COMMON_KWARGS)
|
||||
assert err.parsed_error == "context too long"
|
||||
assert "context too long" in err.body_text
|
||||
|
||||
def test_http_status_error_stream_response_falls_back_to_read(self) -> None:
|
||||
response = httpx.Response(
|
||||
400,
|
||||
stream=httpx.ByteStream(b'{"message": "context too long"}'),
|
||||
request=httpx.Request("POST", "https://api.test.com"),
|
||||
)
|
||||
err = BackendErrorBuilder.build_http_error(
|
||||
error=self._make_http_status_error(response), **self._COMMON_KWARGS
|
||||
)
|
||||
assert err.parsed_error == "context too long"
|
||||
assert "context too long" in err.body_text
|
||||
|
||||
def test_sdk_error_unreadable_response_falls_back_to_str(self) -> None:
|
||||
response = MagicMock(spec=httpx.Response)
|
||||
response.status_code = 400
|
||||
response.reason_phrase = "Bad Request"
|
||||
response.headers = {}
|
||||
type(response).text = property(lambda self: (_ for _ in ()).throw(Exception))
|
||||
response.read.side_effect = Exception("closed")
|
||||
|
||||
sdk_err = SDKError("sdk msg", response, body='{"message": "context too long"}')
|
||||
err = BackendErrorBuilder.build_http_error(error=sdk_err, **self._COMMON_KWARGS)
|
||||
assert err.body_text == '{"message": "context too long"}'
|
||||
assert err.parsed_error == "context too long"
|
||||
|
||||
def test_http_status_error_unreadable_response_falls_back_to_str(self) -> None:
|
||||
response = MagicMock(spec=httpx.Response)
|
||||
response.status_code = 400
|
||||
response.reason_phrase = "Bad Request"
|
||||
response.headers = {}
|
||||
type(response).text = property(lambda self: (_ for _ in ()).throw(Exception))
|
||||
response.read.side_effect = Exception("closed")
|
||||
response.request = httpx.Request("POST", "https://api.test.com")
|
||||
|
||||
http_err = httpx.HTTPStatusError(
|
||||
"http error with details", request=response.request, response=response
|
||||
)
|
||||
err = BackendErrorBuilder.build_http_error(
|
||||
error=http_err, **self._COMMON_KWARGS
|
||||
)
|
||||
assert "http error with details" in err.body_text
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue