Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
248 changes: 248 additions & 0 deletions ontology_sql_eval/gym/model_routing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Route the Auto Ontology arm's LLM calls through NeMo Gym model servers.

Auto Ontology talks to two chat models: a *reasoning* one (SQL generation from
the semantic layer) and a *non-reasoning* one (nearly every other node). Left
alone it calls them directly, so Gym neither serves nor counts any of the ~19
calls a question costs. This module puts a Gym model server in front of each.

Nothing about the models is configured here. Both halves defer to Auto
Ontology's own resolution (``<ROLE>_ENDPOINT`` / ``_MODEL`` / ``_API_KEY``, then
``DEFAULT_MODELS_*``, then its built-in defaults), so a ``.env`` override or a
change to Auto Ontology's defaults carries through without an edit here:

* :func:`launch_env`, run by ``scripts/run_gym_arm.sh`` before Gym starts,
resolves each role and exports it as ``AUTO_ONTOLOGY_GYM_<ROLE>_<FIELD>``. The
arm's config builds one ``inference_provider`` model server per role from
those variables.
* :func:`route_through_gym`, run inside the resources server before Auto
Ontology is imported, repoints ``<ROLE>_ENDPOINT`` at that role's Gym server.
Auto Ontology still resolves the model name itself, which keeps its client
choice (``ChatOpenAI`` vs ``ChatNVIDIA``) the same as a direct run; the Gym
server substitutes its configured model regardless.
* :func:`tag_rollout_requests`, also run inside the resources server, puts the
current rollout's ``/ng-rollout/<id>`` prefix on every request to those
servers. Gym keys token capture by that prefix, so without it the calls are
served but counted against no question.
* :func:`captured_usage` reads one rollout's captured calls back, so ``verify``
can report the question's token totals as rollout fields.

Embedding and reranking are untouched: Gym has no server type for either, so
retrieval still calls its endpoints directly.
"""

from __future__ import annotations

import logging
import os
import shlex
from typing import Any

__all__ = [
"GYM_MODEL_SERVERS",
"captured_usage",
"launch_env",
"route_through_gym",
"tag_rollout_requests",
]

logger = logging.getLogger(__name__)

# Auto Ontology triplet prefix -> the Gym model server that stands in for it.
# The names must match the top-level keys in
# resources_servers/auto_ontology_sql/configs/auto_ontology_sql.yaml.
GYM_MODEL_SERVERS = {
"REASONING": "auto_ontology_reasoning_model",
"NON_REASONING": "auto_ontology_non_reasoning_model",
}

_FIELDS = ("ENDPOINT", "MODEL", "API_KEY")
_LAUNCH_PREFIX = "AUTO_ONTOLOGY_GYM_"

# Root URLs (scheme://host:port) of the servers route_through_gym pointed Auto
# Ontology at. Only requests to these are rewritten by tag_rollout_requests.
_routed_roots: tuple[str, ...] = ()


def launch_env() -> dict[str, str]:
"""Auto Ontology's resolved model triplets, keyed for the arm's Gym config.

Reads the current environment, so ``.env`` must already be loaded.
"""
# Imported here: it pulls in the auto_ontology package, which the rest of
# this package must not require.
from auto_ontology.utils.model_config import resolve

env: dict[str, str] = {}
for role in GYM_MODEL_SERVERS:
for field in _FIELDS:
value = resolve(role, field)
if not value:
raise SystemExit(
f"Auto Ontology resolves no {role}_{field}; set it, or "
f"DEFAULT_MODELS_{field}, in .env"
)
env[f"{_LAUNCH_PREFIX}{role}_{field}"] = value
return env


def route_through_gym() -> None:
"""Point each Auto Ontology role's endpoint at its Gym model server.

Must run before Auto Ontology is imported: its text-to-SQL module builds
both chat clients at import time.
"""
from nemo_gym.server_utils import get_server_url

global _routed_roots
roots = []
for role, server in GYM_MODEL_SERVERS.items():
root = get_server_url(server).rstrip("/")
os.environ[f"{role}_ENDPOINT"] = f"{root}/v1"
roots.append(root)
_routed_roots = tuple(roots)


def with_rollout_prefix(url: str, rollout_id: str | None) -> str:
"""*url* moved under ``/ng-rollout/<rollout_id>`` if it targets a routed server.

Anything else -- embedding and reranking endpoints, an already-prefixed
path, a call made outside any rollout -- is returned unchanged.
"""
from nemo_gym.config_types import ROLLOUT_PATH_PREFIX
from nemo_gym.server_utils import apply_rollout_prefix

for root in _routed_roots:
if not url.startswith(f"{root}/"):
continue
rest = url[len(root) :]
if rest.startswith(f"/{ROLLOUT_PATH_PREFIX}/"):
return url
if not rollout_id:
# Served, but captured against no rollout: the question's token
# count comes up short with nothing else to show for it.
logger.warning(
"Auto Ontology called %s outside any rollout; its tokens will "
"not be counted against a question",
url,
)
return url
return apply_rollout_prefix(root, rollout_id) + rest
return url


def tag_rollout_requests() -> None:
"""Prefix every request to a routed server with the current rollout id.

Patched at the HTTP layer because Auto Ontology builds its chat clients in
many places, some per call, and with two libraries: ``ChatOpenAI`` sends
through httpx, ``ChatNVIDIA`` through requests. The rollout id comes from
Gym's context variable, which ``asyncio.to_thread`` and LangGraph's
executor both copy onto the threads that make the calls. Idempotent.
"""
import httpx
import requests
from nemo_gym.rollout_correlation import current_rollout_id

if getattr(httpx.Client.send, "_gym_rollout_tagged", False):
return

def _tag_httpx(request: httpx.Request) -> None:
tagged = with_rollout_prefix(str(request.url), current_rollout_id())
if tagged != str(request.url):
request.url = httpx.URL(tagged)

def _wrap(send: Any, tag: Any) -> Any:
def wrapped(self: Any, request: Any, *args: Any, **kwargs: Any) -> Any:
tag(request)
return send(self, request, *args, **kwargs)

wrapped._gym_rollout_tagged = True # type: ignore[attr-defined]
return wrapped

def _tag_requests(request: requests.PreparedRequest) -> None:
request.url = with_rollout_prefix(str(request.url), current_rollout_id())

httpx.Client.send = _wrap(httpx.Client.send, _tag_httpx) # type: ignore[method-assign]
httpx.AsyncClient.send = _wrap(httpx.AsyncClient.send, _tag_httpx) # type: ignore[method-assign]
requests.Session.send = _wrap(requests.Session.send, _tag_requests) # type: ignore[method-assign]


def captured_usage(rollout_id: str | None) -> dict[str, int]:
"""Token totals Gym captured for this rollout's Auto Ontology calls.

Returned as flat ints so ``verify`` can put them on the rollout, where Gym
aggregates every numeric field into ``mean/...`` metrics. Gym's own
per-rollout ``ng_model_call_capture`` block is dropped before metrics are
computed, so without this the totals would be in the rollouts file only.

The policy model's call is captured under the same rollout and is left
out: the Auto Ontology arm ignores its output. Empty when observability is
off or nothing was captured.
"""
if not rollout_id:
return {}
from nemo_gym.base_responses_api_model import (
CaptureStore,
model_call_capture_dirs_from_config,
read_available_model_call_records,
)
from nemo_gym.global_config import get_global_config_dict

servers = set(GYM_MODEL_SERVERS.values())
calls = []
for directory in model_call_capture_dirs_from_config(get_global_config_dict()):
records, _ = read_available_model_call_records(
CaptureStore(directory), rollout_id
)
calls += [
c
for c in records
if c.model_ref is not None and c.model_ref.name in servers
]
if not calls:
return {}

def total(attr: str) -> int:
return sum(getattr(c, attr) or 0 for c in calls)

input_tokens = sum(_input_tokens(c) or 0 for c in calls)
output_tokens = total("tokens_out")
return {
"agent_llm_calls": len(calls),
# Calls Gym saw but got no token counts for (an error, an unparsable
# body). Nonzero means the totals below are a floor.
"agent_llm_calls_without_usage": sum(
1 for c in calls if _input_tokens(c) is None
),
"agent_input_tokens": input_tokens,
"agent_cached_input_tokens": total("cached_tokens"),
"agent_output_tokens": output_tokens,
"agent_total_tokens": input_tokens + output_tokens,
}


def _input_tokens(call: Any) -> int | None:
"""One call's prompt tokens, from the provider's own total where it gave one.

Gym's ``tokens_in`` double-counts the inference-api gateway's Claude
responses. Their usage is OpenAI-shaped -- ``prompt_tokens`` already
includes cache writes and reads -- but also carries Anthropic's top-level
``cache_creation_input_tokens``, which Gym takes as the native Anthropic
shape and adds again: 2972 prompt tokens with 2968 written to cache are
recorded as 5940. ``total_tokens`` comes straight from the provider, so
total minus output is the prompt size whichever shape the usage had.
"""
if call.tokens_total is not None and call.tokens_out is not None:
return call.tokens_total - call.tokens_out
return call.tokens_in


if __name__ == "__main__":
# Emitted as shell for `eval`, so the API keys travel through the
# environment rather than argv or a file.
for key, value in launch_env().items():
print(f"export {key}={shlex.quote(value)}")
69 changes: 68 additions & 1 deletion resources_servers/auto_ontology_sql/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,16 @@
set-equality rule, same failure codes -- so a difference in the number can only
come from the SQL.

Auto Ontology's own LLM calls go through two Gym model servers rather than
straight to the endpoint (see ``ontology_sql_eval.gym.model_routing``), each
tagged with the rollout it belongs to. With ``+observability_enabled=true`` Gym
captures them per rollout, and ``verify`` reports the question's token totals.

Three couplings with Auto Ontology this module must respect:

* ``auto_ontology.retrieval.text_to_sql.main`` builds its LLM client and compiles the
graph *at import time*, so the environment is loaded before Auto Ontology imports.
graph *at import time*, so the environment is loaded -- and the endpoints
repointed at Gym -- before Auto Ontology imports.
* Auto Ontology is installed from the parent project's pinned Git dependency.
* ``stream_agent_response`` is a blocking generator, so it runs on a worker
thread under a semaphore rather than on the event loop.
Expand All @@ -39,6 +45,15 @@
# client while its modules are being imported.
load_dotenv()

from ontology_sql_eval.gym.model_routing import ( # noqa: E402
captured_usage,
route_through_gym,
tag_rollout_requests,
)

route_through_gym()
tag_rollout_requests()

from nemo_gym.base_resources_server import ( # noqa: E402
BaseResourcesServerConfig,
BaseVerifyRequest,
Expand All @@ -50,6 +65,7 @@
compute_subset_metrics,
highest_k_metrics,
)
from nemo_gym.rollout_correlation import current_rollout_id # noqa: E402

from auto_ontology.connectors import get_connectors # noqa: E402
from auto_ontology.retrieval.text_to_sql.main import stream_agent_response # noqa: E402
Expand Down Expand Up @@ -202,6 +218,24 @@ class AutoOntologySqlVerifyResponse(BaseVerifyResponse):
# "with prompt inputs" while having had none.
sql_examples_used: int = 0
value_anchors_used: int = 0
# Auto Ontology's own LLM usage for this question, as Gym captured it. None
# rather than 0 when nothing was captured (observability off), so a run
# without capture reports no token metrics instead of a mean of zero.
agent_llm_calls: Optional[int] = None
agent_llm_calls_without_usage: Optional[int] = None
agent_input_tokens: Optional[int] = None
agent_cached_input_tokens: Optional[int] = None
agent_output_tokens: Optional[int] = None
agent_total_tokens: Optional[int] = None


# The per-question token fields, surfaced in key metrics as their means.
_TOKEN_FIELDS = (
"agent_llm_calls",
"agent_input_tokens",
"agent_output_tokens",
"agent_total_tokens",
)


class AutoOntologySqlResourcesServer(SimpleResourcesServer):
Expand Down Expand Up @@ -424,6 +458,7 @@ def _response(
**kwargs,
)

rollout_id = current_rollout_id()
try:
async with self._semaphore:
result = await asyncio.to_thread(
Expand All @@ -437,8 +472,13 @@ def _response(
logger.exception(
"Auto Ontology agent failed task_id=%s: %s", body.task_id, exc
)
# A failed agent has usually spent tokens before failing.
carried.update(await self._usage(rollout_id, body.task_id))
return _response(0.0, FailureCode.UNKNOWN_ERROR, agent_error=str(exc))

# Read by _response at call time, like the counts below.
carried.update(await self._usage(rollout_id, body.task_id))

answer = result["answer"]
sql = str(answer.get("sql_code") or "").strip()
nodes = result["nodes"]
Expand Down Expand Up @@ -501,6 +541,30 @@ def _response(
execution_match=match,
)

async def _usage(
self, rollout_id: Optional[str], task_id: Optional[str]
) -> Dict[str, int]:
"""This rollout's captured token totals, logged per question. Never raises."""
try:
usage = await asyncio.to_thread(captured_usage, rollout_id)
except Exception:
logger.warning(
"Could not read token usage for task_id=%s", task_id, exc_info=True
)
return {}
if usage:
logger.info(
"task_id=%s tokens: %d in (%d cached), %d out, %d total over %d "
"LLM calls",
task_id,
usage["agent_input_tokens"],
usage["agent_cached_input_tokens"],
usage["agent_output_tokens"],
usage["agent_total_tokens"],
usage["agent_llm_calls"],
)
return usage

@staticmethod
def _score_fn(r: dict) -> Dict[str, float]:
return {"accuracy": float(r.get("reward", 0.0) > 0)}
Expand Down Expand Up @@ -541,6 +605,9 @@ def get_key_metrics(self, agent_metrics: Dict[str, Any]) -> Dict[str, Any]:
for name, value in agent_metrics.items()
if name.startswith("health/")
}
for field in _TOKEN_FIELDS:
if f"mean/{field}" in agent_metrics:
key[f"mean/{field}"] = agent_metrics[f"mean/{field}"]
key.update(
highest_k_metrics(
agent_metrics, "pass@1[avg-of-{k}]", score_names=["accuracy"]
Expand Down
Loading
Loading