Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 65 additions & 3 deletions auto_ontology/utils/llm_invoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,62 @@ def _structured_output_kwargs(llm: BaseChatModel) -> dict:
return {}


def _is_chat_nvidia(llm: object) -> bool:
"""Whether *llm* is a ``ChatNVIDIA`` client, including a ``.bind()`` wrapper."""
try:
from langchain_nvidia_ai_endpoints import ChatNVIDIA
except ImportError:
return False
current = llm
for _ in range(_MAX_BINDING_DEPTH):
if isinstance(current, ChatNVIDIA):
return True
bound = getattr(current, "bound", None)
if bound is None or isinstance(bound, (str, bytes, int, float, bool)):
return False
current = bound
return False


def _response_text(response: object) -> str:
content = getattr(response, "content", response)
if isinstance(content, list):
content = "".join(
part if isinstance(part, str) else str(part.get("text", ""))
for part in content
if isinstance(part, (str, dict))
)
return content if isinstance(content, str) else str(content)


def _invoke_guided_json(
llm: BaseChatModel, messages: list[BaseMessage], schema: Type[T]
) -> T | None:
"""Structured call for ``ChatNVIDIA`` with ``guided_json`` at the payload top level.

``ChatNVIDIA.with_structured_output`` (langchain-nvidia-ai-endpoints 1.4.3)
tries OpenAI ``response_format``, then top-level ``guided_json``, then
``nvext.guided_json``, re-raising only the last failure. Against the hosted
``integrate.api.nvidia.com`` endpoint ``response_format`` comes back with
empty content and ``nvext.guided_json`` is rejected with HTTP 400 ("unknown
field `guided_json`"), so every failed attempt cost three requests and
surfaced the misleading ``nvext`` 400. Top-level ``guided_json`` is the shape
the endpoint accepts, and the one self-hosted NIMs are tried with first.

Parsing here instead of in langchain's forgiving parser means a malformed
answer raises ``ValidationError`` and reaches the self-correcting retry,
rather than turning into ``None``. Empty content returns ``None``.
"""
response = llm.bind(guided_json=schema.model_json_schema()).invoke(messages)
text = _response_text(response).strip()
if text.startswith("```"):
text = text.split("\n", 1)[1] if "\n" in text else ""
text = text.rsplit("```", 1)[0].strip()
if not text:
return None
return schema.model_validate_json(text)


def _invoke_text_once(llm: BaseChatModel, prompt: str) -> str:
response = llm.invoke([HumanMessage(content=prompt)])
content = getattr(response, "content", response)
Expand Down Expand Up @@ -598,12 +654,18 @@ def safe_invoke_with_structured_output(
current_messages = _ensure_non_system_message(messages.copy())
schema_name = getattr(schema, "__name__", str(schema))
structured_kwargs = _structured_output_kwargs(llm)
guided_json = _is_chat_nvidia(llm)

for attempt in range(RETRY_MAX_ATTEMPTS):
try:
model_llm = llm.with_structured_output(schema, **structured_kwargs)
with _inflight_guard():
result = model_llm.invoke(_system_messages_first(llm, current_messages))
ordered = _system_messages_first(llm, current_messages)
if guided_json:
with _inflight_guard():
result = _invoke_guided_json(llm, ordered, schema)
else:
model_llm = llm.with_structured_output(schema, **structured_kwargs)
with _inflight_guard():
result = model_llm.invoke(ordered)
except _requests.exceptions.ReadTimeout:
logger.error(
"LLM invoke timed out after %ds on attempt %d/%d for %s",
Expand Down
127 changes: 127 additions & 0 deletions auto_ontology/utils/tests/test_llm_invoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import contextlib
import threading
import time
import warnings
from typing import Any
from unittest.mock import MagicMock

Expand Down Expand Up @@ -497,3 +498,129 @@ def unavailable(**_kwargs: object) -> object:
monkeypatch.setattr(llm_invoke, "get_llm_client", unavailable)

assert llm_invoke.invoke_with_structured_output(primary, [], _Schema) is None


class _GuidedAnswer(llm_invoke.StrictLLMOutputModel):
name: str


class _FakeResponse:
def __init__(self, content: str) -> None:
self._content = content

def json(self) -> dict:
return {
"choices": [
{
"message": {"role": "assistant", "content": self._content},
"finish_reason": "stop",
}
]
}


def _hosted_chat_nvidia(
monkeypatch: pytest.MonkeyPatch, contents: list[str]
) -> tuple[Any, list[dict]]:
"""A real hosted ``ChatNVIDIA`` whose HTTP call returns *contents* in turn."""
from langchain_nvidia_ai_endpoints import ChatNVIDIA
from langchain_nvidia_ai_endpoints._common import _NVIDIABaseClient

# The pinned SDK does not know this model, so constructing the client would
# list models over the network (GET {base_url}/models). Serve an empty list.
monkeypatch.setattr(_NVIDIABaseClient, "available_models", property(lambda _s: []))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
client = ChatNVIDIA(
model="nvidia/nemotron-3.5-lightning-30b-a3b",
api_key="nvapi-test",
base_url="https://integrate.api.nvidia.com/v1",
)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
payloads: list[dict] = []
replies = iter(contents)

def get_req(
_self: object, payload: dict, extra_headers: dict | None = None
) -> _FakeResponse:
payloads.append(payload)
return _FakeResponse(next(replies))

monkeypatch.setattr(type(client._client), "get_req", get_req)
monkeypatch.setattr(llm_invoke.time, "sleep", lambda _s: None)
return client, payloads


def test_chat_nvidia_sends_guided_json_at_the_payload_top_level(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The hosted endpoint rejects ``nvext.guided_json`` (400) and answers
``response_format`` with empty content; only top-level ``guided_json`` works.
"""
client, payloads = _hosted_chat_nvidia(monkeypatch, ['{"name": "x"}'])

result = llm_invoke.safe_invoke_with_structured_output(
client, [HumanMessage(content="q")], _GuidedAnswer
)

assert result == _GuidedAnswer(name="x")
assert len(payloads) == 1
assert payloads[0]["guided_json"] == _GuidedAnswer.model_json_schema()
assert "nvext" not in payloads[0]
assert "response_format" not in payloads[0]


def test_bound_chat_nvidia_keeps_its_bound_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, payloads = _hosted_chat_nvidia(monkeypatch, ['{"name": "x"}'])

result = llm_invoke.safe_invoke_with_structured_output(
client.bind(max_tokens=7), [HumanMessage(content="q")], _GuidedAnswer
)

assert result == _GuidedAnswer(name="x")
assert payloads[0]["max_tokens"] == 7
assert "guided_json" in payloads[0]


def test_chat_nvidia_malformed_answer_reaches_the_validation_retry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
client, payloads = _hosted_chat_nvidia(
monkeypatch, ['{"name": "x", "extra": 1}', '```json\n{"name": "y"}\n```']
)

result = llm_invoke.safe_invoke_with_structured_output(
client, [HumanMessage(content="q")], _GuidedAnswer
)

assert result == _GuidedAnswer(name="y")
assert len(payloads) == 2
assert "did not validate" in payloads[1]["messages"][-1]["content"]


def test_chat_nvidia_empty_answer_is_retried(monkeypatch: pytest.MonkeyPatch) -> None:
client, payloads = _hosted_chat_nvidia(monkeypatch, ["", '{"name": "x"}'])

result = llm_invoke.safe_invoke_with_structured_output(
client, [HumanMessage(content="q")], _GuidedAnswer
)

assert result == _GuidedAnswer(name="x")
assert len(payloads) == 2


def test_other_clients_keep_with_structured_output() -> None:
llm = MagicMock()
llm.model_name = "openai/gpt-5"
llm.with_structured_output.return_value.invoke.return_value = _GuidedAnswer(
name="x"
)

result = llm_invoke.safe_invoke_with_structured_output(
llm, [HumanMessage(content="q")], _GuidedAnswer
)

assert result == _GuidedAnswer(name="x")
llm.with_structured_output.assert_called_once()
assert not llm.bind.called
Loading