Fix issue #762

vibe/core/llm/backend/generic.py

@@ -296,30 +296,10 @@ class GenericBackend:
 
         try:
             res_data, _ = await self._make_request(url, req.body, headers)
-            return adapter.parse_response(res_data, self._provider)
+        except httpx.HTTPStatusError as exc:
+            raise self._handle_error_response(exc, adapter) from exc
 
-        except httpx.HTTPStatusError as e:
+        return adapter.parse_response(res_data, self._provider)
-            raise BackendErrorBuilder.build_http_error(
-                provider=self._provider.name,
-                endpoint=url,
-                error=e,
-                model=model.name,
-                messages=messages,
-                temperature=temperature,
-                has_tools=bool(tools),
-                tool_choice=tool_choice,
-            ) from e
-        except httpx.RequestError as e:
-            raise BackendErrorBuilder.build_request_error(
-                provider=self._provider.name,
-                endpoint=url,
-                error=e,
-                model=model.name,
-                messages=messages,
-                temperature=temperature,
-                has_tools=bool(tools),
-                tool_choice=tool_choice,
-            ) from e
 
     async def complete_streaming(
         self,
@@ -333,6 +313,7 @@ class GenericBackend:
         extra_headers: dict[str, str] | None = None,
         metadata: dict[str, str] | None = None,
     ) -> AsyncGenerator[LLMChunk, None]:
+        DELIM_CHAR = ":"
         api_key = (
             os.getenv(self._provider.api_key_env_var)
             if self._provider.api_key_env_var
@@ -363,81 +344,129 @@ class GenericBackend:
         url = f"{base}{req.endpoint}"
 
         try:
-            async for res_data in self._make_streaming_request(url, req.body, headers):
+            async with (
-                yield adapter.parse_response(res_data, self._provider)
+                await self._get_client().stream(
+                    "POST",
+                    url,
+                    content=req.body,
+                    headers=headers,
+                )
+            ) as response:
+                response.raise_for_status()
+                async for chunk in self._consume_streaming_response(
+                    response, adapter, DELIM_CHAR
+                ):
+                    yield chunk
+        except httpx.HTTPStatusError as exc:
+            raise self._handle_error_response(exc, adapter) from exc
 
-        except httpx.HTTPStatusError as e:
+    async def _consume_streaming_response(
-            raise BackendErrorBuilder.build_http_error(
+        self,
-                provider=self._provider.name,
+        response: httpx.Response,
-                endpoint=url,
+        adapter: APIAdapter,
-                error=e,
+        DELIM_CHAR: str,
-                model=model.name,
+    ) -> AsyncGenerator[LLMChunk, None]:
-                messages=messages,
+        async for line in response.aiter_lines():
-                temperature=temperature,
+            if line.strip() == "" or line.strip().startswith(":"):
-                has_tools=bool(tools),
+                continue
-                tool_choice=tool_choice,
+            if f"{DELIM_CHAR} " not in line:
-            ) from e
+                raise ValueError(
-        except httpx.RequestError as e:
+                    f"Stream chunk improperly formatted. "
-            raise BackendErrorBuilder.build_request_error(
+                    f"Expected `key: value`, received `{line}`"
-                provider=self._provider.name,
+                )
-                endpoint=url,
-                error=e,
-                model=model.name,
-                messages=messages,
-                temperature=temperature,
-                has_tools=bool(tools),
-                tool_choice=tool_choice,
-            ) from e
 
-    class HTTPResponse(NamedTuple):
+            _, value = line.split(DELIM_CHAR, 1)
-        data: dict[str, Any]
+            value = value.strip()
-        headers: dict[str, str]
+            data = json.loads(value)
+
+            if data == "[DONE]":
+                return
+
+            yield adapter.parse_response(data, self._provider)
 
-    @async_retry(tries=3)
     async def _make_request(
-        self, url: str, data: bytes, headers: dict[str, str]
+        self, url: str, body: bytes, headers: dict[str, str]
-    ) -> HTTPResponse:
+    ) -> tuple[dict[str, Any], httpx.Response]:
-        client = self._get_client()
+        response = await self._get_client().post(
-        response = await client.post(url, content=data, headers=headers)
+            url, content=body, headers=headers
+        )
         response.raise_for_status()
+        json_data = response.json()
+        return json_data, response
 
-        response_headers = dict(response.headers.items())
+    def _handle_error_response(
-        response_body = response.json()
+        self, exc: httpx.HTTPStatusError, adapter: APIAdapter
-        return self.HTTPResponse(response_body, response_headers)
+    ) -> BackendErrorBuilder:
+        """Build a BackendErrorBuilder from an HTTP error response."""
+        from vibe.core.llm.exceptions import (
+            BackendError,
+            PayloadSummary,
+            _build_payload_summary_from_request_content,
+        )
 
-    @async_generator_retry(tries=3)
+        request_content = exc.request.content
-    async def _make_streaming_request(
+        payload_summary = (
-        self, url: str, data: bytes, headers: dict[str, str]
+            _build_payload_summary_from_request_content(request_content)
-    ) -> AsyncGenerator[dict[str, Any]]:
+            if request_content
-        client = self._get_client()
+            else PayloadSummary()
-        async with client.stream(
+        )
-            method="POST", url=url, content=data, headers=headers
-        ) as response:
-            if not response.is_success:
-                await response.aread()
-            response.raise_for_status()
-            async for line in response.aiter_lines():
-                if line.strip() == "":
-                    continue
 
-                DELIM_CHAR = ":"
+        # Build the error from the response
-                if f"{DELIM_CHAR} " not in line:
+        error_builder = BackendErrorBuilder(
-                    raise ValueError(
+            provider=self._provider.name,
-                        f"Stream chunk improperly formatted. "
+            model="",
-                        f"Expected `key{DELIM_CHAR} value`, received `{line}`"
+            body=json.dumps(payload_summary.model_dump(), ensure_ascii=False),
-                    )
+            status_code=exc.response.status_code,
-                delim_index = line.find(DELIM_CHAR)
+        )
-                key = line[0:delim_index]
-                value = line[delim_index + 2 :]
 
-                if key != "data":
+        try:
-                    # This might be the case with openrouter, so we just ignore it
+            error_body = exc.response.json()
-                    continue
+            if isinstance(error_body, dict):
-                if value == "[DONE]":
+                error_builder.set_parse_error(error_body)
-                    return
+        except (json.JSONDecodeError, ValueError):
-                yield json.loads(value.strip())
+            error_builder.set_response_text(exc.response.text)
 
-    async def close(self) -> None:
+        return error_builder
-        if self._owns_client and self._client:
+
-            await self._client.aclose()
+    async def _retryable_complete(
-            self._client = None
+        self,
+        model: ModelConfig,
+        messages: Sequence[LLMMessage],
+        temperature: float,
+        tools: list[AvailableTool] | None,
+        max_tokens: int | None,
+        tool_choice: StrToolChoice | AvailableTool | None,
+        **kwargs: Any,
+    ) -> LLMChunk:
+        return await async_retry(
+            self.complete,
+            model=model,
+            messages=messages,
+            temperature=temperature,
+            tools=tools,
+            max_tokens=max_tokens,
+            tool_choice=tool_choice,
+            **kwargs,
+        )
+
+    async def _retryable_complete_streaming(
+        self,
+        model: ModelConfig,
+        messages: Sequence[LLMMessage],
+        temperature: float,
+        tools: list[AvailableTool] | None,
+        max_tokens: int | None,
+        tool_choice: StrToolChoice | AvailableTool | None,
+        **kwargs: Any,
+    ) -> AsyncGenerator[LLMChunk, None]:
+        async for chunk in async_generator_retry(
+            self.complete_streaming,
+            model=model,
+            messages=messages,
+            temperature=temperature,
+            tools=tools,
+            max_tokens=max_tokens,
+            tool_choice=tool_choice,
+            **kwargs,
+        ):
+            yield chunk