From 8f4eef14ee88cd23d36e375c61d9b64c79bcea00 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:31:22 -0600 Subject: [PATCH 01/10] Build the catalog schema map once per catalog state SQL validation read every column of every database and rebuilt the Schema objects before each parse, once per SQL attempt and up to eight times per question. The cost is linear in catalog size: 31 ms at 440 columns, 1.5 s at 50,000, 4.6 s at 150,000, per attempt. Keep one map for the process and rebuild it only when a fingerprint of the rows changes. The fingerprint is one aggregate, checked on every validation, so a table ingested a moment ago is still seen by the next parse. The map is stored under the fingerprint returned with its rows by the same statement, so a commit between two reads cannot mislabel it. Parsing against the map is serialised, and each parse starts with no nodes left from the previous one, so a lookup resolves as it did against a freshly built schema. The extractor no longer walks every column of the catalog per statement either: it reads a per-table index kept on each Schema, and reuses the sqlglot MappingSchema per dialect. Per validation this is now 2 ms, 5 ms and 12 ms at the sizes above. --- auto_ontology/catalog/model/schema.py | 59 +++- .../catalog/parsers/sqlglot_extractor.py | 108 ++++++- auto_ontology/dal/datasources.py | 89 ++++++ auto_ontology/dal/tests/dal_surface.json | 2 + .../dal/tests/test_datasources_semantic.py | 78 +++++ .../retrieval/data_access/graph_schemas.py | 120 +++++++- .../data_access/tests/test_catalog_schemas.py | 282 ++++++++++++++++++ .../agents/sql_parse_validation.py | 21 +- .../tests/test_intent_validation.py | 21 +- .../text_to_sql/tests/test_numeric_types.py | 3 +- 10 files changed, 734 insertions(+), 49 deletions(-) create mode 100644 auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py diff --git a/auto_ontology/catalog/model/schema.py b/auto_ontology/catalog/model/schema.py index 73f82a280..349ca940f 100644 --- a/auto_ontology/catalog/model/schema.py +++ b/auto_ontology/catalog/model/schema.py @@ -92,12 +92,15 @@ def __init__( self.columns_df.set_index("id")["full_name"].to_dict() ) - # This is for improving is_column_in_table + # This is for improving is_column_in_table. Read column-wise: one + # `iterrows()` pass builds a Series per row, which at catalog + # scale was half the cost of constructing a Schema. self.columns_map = {} - for _, row in self.columns_df.iterrows(): - if row["table_name_lower"] not in self.columns_map: - self.columns_map[row["table_name_lower"]] = {} - self.columns_map[row["table_name_lower"]][row["column_name_lower"]] = ( + for table_name_lower, column_name_lower in zip( + self.columns_df["table_name_lower"], + self.columns_df["column_name_lower"], + ): + self.columns_map.setdefault(table_name_lower, {})[column_name_lower] = ( True ) @@ -151,6 +154,52 @@ def reset_columns_props(self): axis=1, ) + def column_names_by_table(self) -> dict[str, tuple[str, ...]]: + """Lower-cased column names per lower-cased table name, in row order. + + This is what SQL extraction needs from the catalog on every parse: the + column set of each table, to hand to sqlglot's ``qualify`` and to + attribute bare column references. Reading it off ``columns_df`` row by + row costs a pandas Series per column of the *whole* schema, per parse, + which made a parse scale with the catalog instead of with the query. + It is derived once and kept for as long as ``columns_df`` is the same + frame with the same number of rows. + + Names are lower-cased with ``str(...).lower()`` and nothing else, on + purpose: unlike ``columns_map`` they keep any surrounding quotes, which + is the spelling the extractor has always matched against. + """ + df = self.columns_df + if df is None or df.empty: + return {} + # The frame itself is held, not its id(): an id can be handed to a new + # object once the old one is collected, and would then vouch for an + # index read from a frame that no longer exists. + cached = getattr(self, "_column_names_by_table", None) + if cached is not None and cached[0] is df and cached[1] == len(df): + return cached[2] + grouped: dict[str, list[str]] = {} + for table_name, column_name in zip(df["table_name"], df["column_name"]): + grouped.setdefault(str(table_name).lower(), []).append( + str(column_name).lower() + ) + index = {table: tuple(columns) for table, columns in grouped.items()} + self._column_names_by_table = (df, len(df), index) + return index + + def forget_nodes(self) -> None: + """Drop the table and column nodes built by earlier lookups. + + ``get_table_node`` and ``get_column_node`` build a node on first use and + keep it. A schema that is parsed against repeatedly calls this between + parses so that each one resolves names as the first did; the frames, + ``columns_map`` and ``id_to_node`` it resolves them from are untouched. + """ + self.table_nodes = {} + self.column_nodes = {} + self.tables_to_columns = {} + self.tables_columns_pos = {} + def get_schema_name(self): return self.schema_name diff --git a/auto_ontology/catalog/parsers/sqlglot_extractor.py b/auto_ontology/catalog/parsers/sqlglot_extractor.py index e4372e195..ca290b68d 100644 --- a/auto_ontology/catalog/parsers/sqlglot_extractor.py +++ b/auto_ontology/catalog/parsers/sqlglot_extractor.py @@ -27,11 +27,13 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import Any import sqlglot from sqlglot import exp from sqlglot.errors import ParseError, TokenError from sqlglot.optimizer.qualify import qualify +from sqlglot.schema import MappingSchema class SQLSyntaxError(ValueError): @@ -151,6 +153,54 @@ def _alias_to_table_map( return mapping +class SchemaCatalog(dict): + """A ``{schema_name: Schema}`` map that is parsed against more than once. + + Extraction derives two things from the catalog that depend on the catalog + alone, not on the SQL: the flat ``{table: {column: type}}`` mapping + ``qualify`` reads, and the ``MappingSchema`` sqlglot normalises that + mapping into for one dialect. Both are linear in the number of columns in + the *whole* catalog, so rebuilding them per statement makes every parse + pay for every table nobody asked about. + + A plain ``dict`` has nowhere to keep them. Wrapping the map in this class + is how a caller says the same catalog will be reused, and that it will not + add, drop or swap a schema in place afterwards -- build a new + ``SchemaCatalog`` for a changed catalog instead. A plain ``dict`` still + works everywhere and is simply derived from each time. + + Not safe for concurrent parses on its own: these caches, like the lazily + built nodes on each ``Schema``, are filled on first use. Callers sharing + one across threads serialise the parse. + """ + + __slots__ = ("_qualify_columns", "_qualify_schemas") + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._qualify_columns: dict[str, dict[str, str]] | None = None + self._qualify_schemas: dict[str, MappingSchema] = {} + + +def _schema_column_index(schema: Any) -> dict[str, tuple[str, ...]]: + """``{table: (column, ...)}`` for one catalog schema, names lower-cased. + + A :class:`~auto_ontology.catalog.model.Schema` keeps this itself; anything + else that merely looks like one (a ``columns_df`` and nothing more) has it + derived here, the same way. + """ + accessor = getattr(schema, "column_names_by_table", None) + if callable(accessor): + return accessor() + df = getattr(schema, "columns_df", None) + if df is None or df.empty: + return {} + grouped: dict[str, list[str]] = {} + for table_name, column_name in zip(df["table_name"], df["column_name"]): + grouped.setdefault(str(table_name).lower(), []).append(str(column_name).lower()) + return {table: tuple(columns) for table, columns in grouped.items()} + + def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]: """Build a flat ``{table: {col: "TEXT"}}`` dict for sqlglot's qualify pass. @@ -161,18 +211,43 @@ def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]: Multi-schema disambiguation is handled separately via ``alias_map`` and ``source_table_names`` (which use fully-qualified keys). """ + if isinstance(all_schemas, SchemaCatalog) and ( + all_schemas._qualify_columns is not None + ): + return all_schemas._qualify_columns + schema: dict[str, dict[str, str]] = {} for s in all_schemas.values(): - df = s.columns_df - if df is None or df.empty: - continue - for _, row in df.iterrows(): - tbl = str(row["table_name"]).lower() - col = str(row["column_name"]).lower() - schema.setdefault(tbl, {})[col] = "TEXT" + for tbl, columns in _schema_column_index(s).items(): + table_columns = schema.setdefault(tbl, {}) + for col in columns: + table_columns[col] = "TEXT" + + if isinstance(all_schemas, SchemaCatalog): + all_schemas._qualify_columns = schema return schema +def _qualify_schema( + all_schemas: dict, schema_dict: dict[str, dict[str, str]], dialect: str +) -> "MappingSchema | dict[str, dict[str, str]]": + """What to hand ``qualify`` as its schema for *dialect*. + + ``qualify`` turns a mapping into ``MappingSchema(mapping, dialect=dialect)`` + before it does anything else, and passes an existing schema object through + untouched. For a :class:`SchemaCatalog` that object is built once per + dialect here, by that exact call, and reused; sqlglot only reads from it. + Everything else gets the mapping, as before. + """ + if not isinstance(all_schemas, SchemaCatalog) or not schema_dict: + return schema_dict + cached = all_schemas._qualify_schemas.get(dialect) + if cached is None: + cached = MappingSchema(schema_dict, dialect=dialect) + all_schemas._qualify_schemas[dialect] = cached + return cached + + _BINARY_CMP = (exp.EQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.NEQ) _OP_SYMBOL: dict[type, str] = { @@ -692,7 +767,7 @@ def extract_tables_and_columns( qualified = qualify( statement, dialect=dialect, - schema=schema_dict, + schema=_qualify_schema(all_schemas, schema_dict, dialect), qualify_columns=True, validate_qualify_columns=False, expand_stars=bool(schema_dict), @@ -718,13 +793,11 @@ def extract_tables_and_columns( if unresolved and all_schemas: col_to_tables: dict[str, list[str]] = {} for schema_name, s in all_schemas.items(): - df = s.columns_df - if df is None or df.empty: - continue skey = schema_name.lower() - for _, row in df.iterrows(): - col_n = str(row["column_name"]).lower() - tbl_n = str(row["table_name"]).lower() + # Per table, not per column: which source table a catalog table + # stands for is decided by its name alone, so only the columns of + # tables this query actually reads are ever walked. + for tbl_n, columns in _schema_column_index(s).items(): # Try the qualified name first (schema.table); fall back to bare # table name for SQL that doesn't prefix tables with a schema. qualified_name = f"{skey}.{tbl_n}" @@ -733,8 +806,11 @@ def extract_tables_and_columns( if qualified_name in source_table_names else (tbl_n if tbl_n in source_table_names else None) ) - if matched and matched not in col_to_tables.get(col_n, []): - col_to_tables.setdefault(col_n, []).append(matched) + if not matched: + continue + for col_n in columns: + if matched not in col_to_tables.get(col_n, []): + col_to_tables.setdefault(col_n, []).append(matched) for col_name in unresolved: candidates = col_to_tables.get(col_name, []) diff --git a/auto_ontology/dal/datasources.py b/auto_ontology/dal/datasources.py index e8bffaa8f..2060a69e9 100644 --- a/auto_ontology/dal/datasources.py +++ b/auto_ontology/dal/datasources.py @@ -24,9 +24,11 @@ import pandas as pd from sqlalchemy import ( ColumnElement, + Numeric, Text, and_, case, + cast, column, distinct, func, @@ -251,6 +253,93 @@ def fetch_schemas_by_ids( return [dict(r) for r in store().query_read(statement)] +def _schema_row_hash() -> ColumnElement: + """One 64-bit hash per row of the ``fetch_schemas_by_ids`` result. + + Over exactly the seven fields that query selects, and nothing else: a + description edit or a new sample value leaves it alone, while a renamed + column, a retyped one, or a table added, dropped or moved does not. + + ``data_type`` goes last because it is the only nullable field and + ``concat_ws`` skips nulls: at the end, a missing type shortens the string + rather than shifting another field into its place. + """ + return func.hashtextextended( + func.concat_ws( + # The unit separator, which no identifier contains, so two rows + # cannot hash alike by splitting the same text at different points. + literal("\x1f"), + s.catalog_column.c.id, + s.catalog_column.c.name, + s.catalog_table.c.id, + s.catalog_table.c.name, + s.catalog_schema.c.name, + s.catalog_database.c.name, + s.catalog_column.c.data_type, + ), + 0, + ) + + +def _schema_fingerprint(row_count: int, digest: Any) -> str: + """How a row count and a sum of row hashes are written down. + + One place, because the two functions below must agree to the character: + one gets the sum from Postgres as a ``Decimal``, the other adds the hashes + up itself. + """ + return f"{int(row_count)}:{int(digest)}" + + +def fetch_catalog_schema_fingerprint() -> str: + """A digest that changes whenever ``fetch_schemas_by_ids()`` would. + + Lets a caller keep what it built from the whole-catalog rows and learn, + for the price of one aggregate, whether it is still current. The rows are + hashed where they live, so nothing but the answer crosses the wire. + + The sum makes it independent of row order, and the count keeps an added + row from being cancelled by a removed one whose hashes happen to offset. + Compare it only with the fingerprint + :func:`fetch_catalog_schema_snapshot` returned alongside the rows. + """ + statement = select( + func.count().label("row_count"), + # Numeric: summing bigint hashes overflows bigint long before the + # catalog is large. + func.coalesce(func.sum(cast(_schema_row_hash(), Numeric)), 0).label("digest"), + ).select_from(_catalog_join()) + + row = store().query_read(statement)[0] + return _schema_fingerprint(row["row_count"], row["digest"]) + + +def fetch_catalog_schema_snapshot() -> tuple[str, list[dict[str, str]]]: + """Every row ``fetch_schemas_by_ids()`` returns, with their fingerprint. + + The fingerprint is computed from the rows returned, which one statement + read from one snapshot of the database. Reading the fingerprint and the + rows separately cannot promise that: a commit landing between the two + pairs a fingerprint with rows it does not describe, and because the + fingerprint is a function of content, a catalog that later returns to the + earlier state would match it again and vouch for the wrong rows. + """ + statement = select( + s.catalog_column.c.name.label("column_name"), + s.catalog_column.c.id.label("column_id"), + s.catalog_table.c.name.label("table_name"), + s.catalog_table.c.id.label("table_id"), + s.catalog_database.c.name.label("database_name"), + s.catalog_schema.c.name.label("table_schema"), + s.catalog_column.c.data_type, + _schema_row_hash().label("row_hash"), + ).select_from(_catalog_join()) + + rows = [dict(r) for r in store().query_read(statement)] + digest = sum(row.pop("row_hash") for row in rows) + return _schema_fingerprint(len(rows), digest), rows + + # --------------------------------------------------------------------------- # Table # --------------------------------------------------------------------------- diff --git a/auto_ontology/dal/tests/dal_surface.json b/auto_ontology/dal/tests/dal_surface.json index 58d8824ac..159a938d5 100644 --- a/auto_ontology/dal/tests/dal_surface.json +++ b/auto_ontology/dal/tests/dal_surface.json @@ -52,6 +52,8 @@ "fetch_all_schema_ids": "() -> 'list[str]'", "fetch_all_tables_without_term": "(database_name: 'str | None' = None) -> 'list[dict[str, Any]]'", "fetch_bridge_table_candidates": "(database_name: 'str') -> 'list[dict[str, Any]]'", + "fetch_catalog_schema_fingerprint": "() -> 'str'", + "fetch_catalog_schema_snapshot": "() -> 'tuple[str, list[dict[str, str]]]'", "fetch_col_table_contexts": "(col_ids: 'list[str]') -> 'dict[str, dict[str, str]]'", "fetch_columns_for_table": "(table_id: 'str', *, skip: 'int' = 0, limit: 'int | None' = None) -> 'dict[str, Any] | None'", "fetch_databases": "(zone_ids: 'list[str] | None' = None) -> 'list[dict[str, Any]]'", diff --git a/auto_ontology/dal/tests/test_datasources_semantic.py b/auto_ontology/dal/tests/test_datasources_semantic.py index c546e50fd..89e4525d6 100644 --- a/auto_ontology/dal/tests/test_datasources_semantic.py +++ b/auto_ontology/dal/tests/test_datasources_semantic.py @@ -746,3 +746,81 @@ def test_an_fk_pointing_outside_the_catalog_disqualifies(world) -> None: assert "tag_link" not in [ r["table_name"] for r in d.fetch_bridge_table_candidates(world.prefix) ] + + +# --------------------------------------------------------------------------- +# fetch_catalog_schema_fingerprint +# --------------------------------------------------------------------------- + + +def _row_set(rows: list[dict]) -> set[tuple]: + return {tuple(sorted(row.items(), key=lambda item: item[0])) for row in rows} + + +def test_fingerprint_tracks_exactly_what_the_schema_rows_contain(world: World) -> None: + """It must move with every field ``fetch_schemas_by_ids`` returns, and no other. + + Whatever is built from those rows is kept for as long as the fingerprint + holds, so a change it misses is a stale catalog served to SQL validation. + """ + seen = [d.fetch_catalog_schema_fingerprint()] + + def changed() -> bool: + seen.append(d.fetch_catalog_schema_fingerprint()) + # The fingerprint that comes back with the rows is the one a cached + # map is filed under, so it has to be the same string, every time. + snapshot_fingerprint, rows = d.fetch_catalog_schema_snapshot() + assert snapshot_fingerprint == seen[-1] + assert _row_set(rows) == _row_set(d.fetch_schemas_by_ids()) + return seen[-1] != seen[-2] + + assert not changed(), "two reads of an untouched catalog must agree" + + world.table("refunds", table_type="base table") + assert not changed(), "a table with no columns contributes no rows" + + refund_id = world.column("refunds", "refund_id", 1) + assert changed(), "an added column" + + store().query_write( + s.catalog_column.update() + .where(s.catalog_column.c.id == refund_id) + .values(description="Refund identifier", sample_values='["r1"]') + ) + assert not changed(), "a description or sample values are not in the rows" + + store().query_write( + s.catalog_column.update() + .where(s.catalog_column.c.id == refund_id) + .values(data_type="bigint") + ) + assert changed(), "a type going from NULL to a value" + + store().query_write( + s.catalog_column.update() + .where(s.catalog_column.c.id == refund_id) + .values(name="refund_key") + ) + assert changed(), "a renamed column" + + store().query_write( + s.catalog_table.update() + .where(s.catalog_table.c.id == world.tables["refunds"]) + .values(name="returns") + ) + assert changed(), "a renamed table" + + store().query_write( + s.catalog_schema.update() + .where(s.catalog_schema.c.id == world.schema) + .values(name="store") + ) + assert changed(), "a renamed schema" + + store().query_write( + s.catalog_column.delete().where(s.catalog_column.c.id == refund_id) + ) + assert changed(), "a dropped column" + assert len(set(seen)) == len(seen) - 3, ( + "each change gave a fingerprint not seen before" + ) diff --git a/auto_ontology/retrieval/data_access/graph_schemas.py b/auto_ontology/retrieval/data_access/graph_schemas.py index bbfa96552..6bcd04583 100644 --- a/auto_ontology/retrieval/data_access/graph_schemas.py +++ b/auto_ontology/retrieval/data_access/graph_schemas.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging +import threading import time import pandas as pd @@ -22,9 +23,12 @@ from auto_ontology.catalog.model import CatalogNode from auto_ontology.catalog.constants import Labels from auto_ontology.catalog.model import Schema +from auto_ontology.catalog.parsers.sqlglot_extractor import SchemaCatalog from auto_ontology.dal.datasources import ( fetch_all_schema_ids, + fetch_catalog_schema_fingerprint, + fetch_catalog_schema_snapshot, fetch_schemas_by_ids, fetch_item_by_id, ) @@ -32,11 +36,117 @@ logger = logging.getLogger(__name__) __all__ = [ + "catalog_schemas_lock", "fetch_all_schema_ids", "fetch_item_by_id", + "forget_parsed_nodes", + "get_catalog_schemas", "get_schemas_by_ids", + "reset_catalog_schemas", ] +#: Held while parsing against what :func:`get_catalog_schemas` returns. +#: +#: That map is shared by every caller in the process, and both it and the +#: ``Schema`` objects in it fill caches on first use (table and column nodes, +#: the mapping ``qualify`` reads). Two threads filling one at once would each +#: build their own copy of a node and disagree about which is *the* node for a +#: table. A parse against the shared map takes milliseconds, so one lock around +#: it costs nothing next to the model call that follows. Reentrant, so a caller +#: already holding it can still ask for the map. +catalog_schemas_lock = threading.RLock() + +# The whole-catalog map and the fingerprint of the rows it was built from. +_catalog_snapshot: tuple[str, SchemaCatalog] | None = None + + +def reset_catalog_schemas() -> None: + """Forget the cached catalog map, so the next caller builds a fresh one.""" + global _catalog_snapshot + with catalog_schemas_lock: + _catalog_snapshot = None + + +def forget_parsed_nodes(schemas: dict) -> None: + """Start a parse against the shared map with no nodes left from the last one. + + A ``Schema`` builds a table or column node the first time it is asked for + one and answers from memory after that. Kept across parses, that memory + makes a lookup depend on what was parsed before it: a column whose stored + name carries surrounding quotes fails its first lookup and succeeds on + every later one, so the columns validation reports for the same SQL would + differ between the first question and the second. + + Every validation used to get schemas nobody had parsed against. Dropping + the nodes gives each parse the same starting point and keeps what is + expensive: the frames, the column index and the id map never change after + the build. Only the map from :func:`get_catalog_schemas` is touched; a + map built for one call has no earlier parse to forget. + """ + if not isinstance(schemas, SchemaCatalog): + return + for schema in schemas.values(): + schema.forget_nodes() + + +def get_catalog_schemas() -> dict: + """The schema map for the whole catalog, rebuilt only when the catalog changes. + + Equivalent to ``get_schemas_by_ids(fetch_all_schema_ids())``, which is what + SQL validation used to run before every parse. That reads every column of + every database and turns the rows into ``Schema`` objects; it is linear in + the size of the catalog, and a question is validated once per SQL attempt, + up to eight times. Measured without a database in the way: 30 ms for a + 440-column catalog, 1.5 s at 50,000 columns, 4.6 s at 150,000 -- per + attempt, on the interpreter lock, so concurrent questions queued behind it. + + Here the rows are read only when + :func:`~auto_ontology.dal.datasources.fetch_catalog_schema_fingerprint` + differs from the fingerprint of the rows the cached map was built from. + The check is a single aggregate, and it is made on every call: a table + ingested a moment ago is visible to the very next validation, exactly as + it was before. + + The map is stored under the fingerprint that came back *with its rows*, + from the same statement, never under the one read beforehand. A commit + between two separate reads would otherwise file new rows under an old + fingerprint, and a catalog that later returned to that old state would be + served the wrong map for as long as it stayed there. + + When the fingerprint cannot be read at all, the map is built as it always + was and not kept. Anything raised while building propagates and nothing is + cached, so an empty or unreachable catalog fails the same way it did. + + Hold :data:`catalog_schemas_lock` while parsing against the result, and + call :func:`forget_parsed_nodes` first. + """ + global _catalog_snapshot + + try: + fingerprint = fetch_catalog_schema_fingerprint() + except Exception: + logger.warning( + "Catalog fingerprint unavailable; building the schema map uncached", + exc_info=True, + ) + return get_schemas_by_ids(fetch_all_schema_ids()) + + with catalog_schemas_lock: + snapshot = _catalog_snapshot + if snapshot is not None and snapshot[0] == fingerprint: + return snapshot[1] + started = time.perf_counter() + rows_fingerprint, rows = fetch_catalog_schema_snapshot() + schemas = SchemaCatalog(_schemas_from_rows(rows)) + _catalog_snapshot = (rows_fingerprint, schemas) + logger.info( + "Catalog schema map %s in %.2fs (%d schema(s))", + "built" if snapshot is None else "rebuilt after a catalog change", + time.perf_counter() - started, + len(schemas), + ) + return schemas + def get_schemas_by_ids(relevant_schemas_ids: list | None = None) -> dict: """Assemble :class:`Schema` objects from catalog data. @@ -47,6 +157,12 @@ def get_schemas_by_ids(relevant_schemas_ids: list | None = None) -> dict: before_get_all = time.time() data_array = fetch_schemas_by_ids(relevant_schemas_ids) logger.info(f"time took to get all data from graph: {time.time() - before_get_all}") + return _schemas_from_rows(data_array) + + +def _schemas_from_rows(data_array: list[dict[str, str]]) -> dict: + """Turn ``fetch_schemas_by_ids``-shaped rows into the ``Schema`` map.""" + before_get_all = time.time() data_df = pd.DataFrame(data_array) dbs = list(data_df["database_name"].unique()) @@ -101,5 +217,7 @@ def get_schemas_by_ids(relevant_schemas_ids: list | None = None) -> dict: logger.info( f"total time it took to create all schemas nodes: {time.time() - before_modify_all}" ) - logger.info(f"total time for get_schemas_by_ids(): {time.time() - before_get_all}") + logger.info( + f"total time to build schemas from rows: {time.time() - before_get_all}" + ) return all_schemas diff --git a/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py b/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py new file mode 100644 index 000000000..fcd07e36e --- /dev/null +++ b/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The whole-catalog schema map is built once per catalog state, never stale. + +SQL validation parses every attempt against it. Rebuilding it each time made a +validation as slow as the catalog is large; keeping it is only safe if a +changed catalog is always noticed, and if what a parse returns from the kept +map is what it returned from a fresh one. +""" + +from __future__ import annotations + +import uuid +from typing import Any + +import pytest + +from auto_ontology.catalog.parsers.sqlglot_extractor import ( + SchemaCatalog, + extract_tables_and_columns, +) +from auto_ontology.catalog.sql_parse import parse_query_single +from auto_ontology.retrieval.data_access import graph_schemas + + +def _rows(tables: dict[str, list[str]], schema: str = "shop") -> list[dict[str, str]]: + """Catalog rows in the shape ``fetch_schemas_by_ids`` returns.""" + rows = [] + for table_name, columns in tables.items(): + table_id = str(uuid.uuid4()) + for column_name in columns: + rows.append( + { + "column_name": column_name, + "column_id": str(uuid.uuid4()), + "table_name": table_name, + "table_id": table_id, + "database_name": "db", + "table_schema": schema, + "data_type": "text", + } + ) + return rows + + +class _Catalog: + """A stand-in for the catalog tables, counting how often it is read. + + ``fingerprint`` is what the cheap check reports. ``rows_fingerprint`` is + what comes back with the rows, and is the same thing unless a test is + simulating a commit that landed between the two reads. + """ + + def __init__(self, tables: dict[str, list[str]]) -> None: + self.rows = _rows(tables) + self.fingerprint = "1" + self.rows_fingerprint: str | None = None + self.reads = 0 + + def fetch_rows(self, schema_ids: list | None = None) -> list[dict[str, str]]: + self.reads += 1 + return self.rows + + def fetch_snapshot(self) -> tuple[str, list[dict[str, str]]]: + self.reads += 1 + return self.rows_fingerprint or self.fingerprint, self.rows + + def install(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(graph_schemas, "fetch_schemas_by_ids", self.fetch_rows) + monkeypatch.setattr(graph_schemas, "fetch_all_schema_ids", lambda: []) + monkeypatch.setattr( + graph_schemas, "fetch_catalog_schema_snapshot", self.fetch_snapshot + ) + monkeypatch.setattr( + graph_schemas, + "fetch_catalog_schema_fingerprint", + lambda: self.fingerprint, + ) + + +@pytest.fixture(autouse=True) +def _fresh_snapshot() -> Any: + graph_schemas.reset_catalog_schemas() + yield + graph_schemas.reset_catalog_schemas() + + +@pytest.fixture +def catalog(monkeypatch: pytest.MonkeyPatch) -> _Catalog: + catalog = _Catalog( + { + "orders": ["order_id", "customer_id", "total"], + "customers": ["customer_id", "name"], + } + ) + catalog.install(monkeypatch) + return catalog + + +def test_unchanged_catalog_is_read_once(catalog: _Catalog) -> None: + first = graph_schemas.get_catalog_schemas() + second = graph_schemas.get_catalog_schemas() + + assert second is first + assert catalog.reads == 1 + + +def test_changed_catalog_is_rebuilt_on_the_next_call(catalog: _Catalog) -> None: + before = graph_schemas.get_catalog_schemas() + assert not before["shop"].table_exists("refunds") + + catalog.rows = catalog.rows + _rows({"refunds": ["refund_id"]}) + catalog.fingerprint = "2" + after = graph_schemas.get_catalog_schemas() + + assert after is not before + assert after["shop"].table_exists("refunds") + assert catalog.reads == 2 + + +def test_map_is_filed_under_the_fingerprint_of_its_own_rows(catalog: _Catalog) -> None: + """A commit between the check and the read must not mislabel the map. + + The check sees the catalog without ``refunds``; by the time the rows are + read, ``refunds`` has been committed. Filed under the fingerprint from the + check, that map would be served again the moment ``refunds`` was dropped + and the catalog's fingerprint returned to the earlier value -- validating + SQL against a table that no longer exists. + """ + without_refunds = catalog.rows + catalog.fingerprint = "without-refunds" + catalog.rows = without_refunds + _rows({"refunds": ["refund_id"]}) + catalog.rows_fingerprint = "with-refunds" + + assert graph_schemas.get_catalog_schemas()["shop"].table_exists("refunds") + + # refunds is dropped again: the catalog is back to the state the first + # check saw, and reports the same fingerprint it did then. + catalog.rows = without_refunds + catalog.rows_fingerprint = None + + assert not graph_schemas.get_catalog_schemas()["shop"].table_exists("refunds") + + +def test_unreadable_fingerprint_builds_fresh_every_time( + catalog: _Catalog, monkeypatch: pytest.MonkeyPatch +) -> None: + def unavailable() -> str: + raise RuntimeError("function hashtextextended does not exist") + + monkeypatch.setattr(graph_schemas, "fetch_catalog_schema_fingerprint", unavailable) + + first = graph_schemas.get_catalog_schemas() + second = graph_schemas.get_catalog_schemas() + + assert first["shop"].table_exists("orders") + assert second is not first + assert catalog.reads == 2 + + +def test_a_failed_build_is_not_kept( + catalog: _Catalog, monkeypatch: pytest.MonkeyPatch +) -> None: + """An empty catalog raises, as it always did, and leaves nothing behind.""" + populated = catalog.rows + catalog.rows = [] + with pytest.raises(KeyError): + graph_schemas.get_catalog_schemas() + + # Same fingerprint on purpose: had the failure been cached, this would + # return whatever was stored instead of building. + catalog.rows = populated + assert graph_schemas.get_catalog_schemas()["shop"].table_exists("orders") + + +_QUERIES = [ + "SELECT name FROM customers", + "SELECT * FROM shop.orders", + "SELECT o.total, c.name FROM orders o JOIN customers c " + "ON o.customer_id = c.customer_id WHERE total > 10", + "SELECT customer_id FROM orders JOIN customers USING (customer_id)", + "WITH big AS (SELECT customer_id, total FROM orders WHERE total > 100) " + "SELECT name, total FROM big JOIN customers " + "ON big.customer_id = customers.customer_id", + "SELECT order_id FROM orders UNION ALL SELECT customer_id FROM customers", + "SELECT missing_column FROM orders, customers WHERE name = 'x'", + "SELECT 1 FROM not_in_the_catalog", +] + + +def _extraction(sql: str, schemas: dict, dialect: str) -> Any: + result = extract_tables_and_columns( + sql=sql, dialects=[dialect], all_schemas=schemas + ) + return ( + {name: (sorted(m.columns), m.schema_name) for name, m in result.tables.items()}, + result.ast_node_count, + result.joins, + result.unions, + ) + + +@pytest.mark.parametrize("dialect", ["postgres", "snowflake", "databricks"]) +@pytest.mark.parametrize("sql", _QUERIES) +def test_shared_map_extracts_what_a_fresh_map_does( + catalog: _Catalog, sql: str, dialect: str +) -> None: + """Twice against the shared map, so the second run reads its caches.""" + fresh = graph_schemas.get_schemas_by_ids([]) + shared = graph_schemas.get_catalog_schemas() + assert isinstance(shared, SchemaCatalog) + + expected = _extraction(sql, fresh, dialect) + + assert _extraction(sql, shared, dialect) == expected + assert _extraction(sql, shared, dialect) == expected + + +def test_shared_map_resolves_the_same_catalog_ids(catalog: _Catalog) -> None: + sql = "SELECT o.total, c.name FROM orders o JOIN customers c ON o.customer_id = c.customer_id" + fresh = parse_query_single( + sql=sql, schemas=graph_schemas.get_schemas_by_ids([]), dialects=["postgres"] + ) + shared_schemas = graph_schemas.get_catalog_schemas() + + for _ in range(2): + with graph_schemas.catalog_schemas_lock: + shared = parse_query_single( + sql=sql, schemas=shared_schemas, dialects=["postgres"] + ) + assert sorted(shared.get_tables_ids()) == sorted(fresh.get_tables_ids()) + assert sorted(shared.get_column_ids()) == sorted(fresh.get_column_ids()) + + +def test_a_parse_does_not_see_nodes_an_earlier_parse_built( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The same SQL must report the same columns on the first parse and the tenth. + + A column stored with its quotes fails the lookup that builds its node and + is left out, but the half-built node stays behind; asked again, the schema + answers from it and the column appears. With one map shared by every + validation, what a question was told would depend on the questions before. + """ + catalog = _Catalog({"orders": ["order_id", '"Total"']}) + catalog.install(monkeypatch) + sql = "SELECT order_id, total FROM orders" + + def parsed_columns(schemas: dict) -> list[str]: + with graph_schemas.catalog_schemas_lock: + graph_schemas.forget_parsed_nodes(schemas) + query = parse_query_single(sql=sql, schemas=schemas, dialects=["postgres"]) + return sorted(query.get_column_ids()) + + fresh = parsed_columns(graph_schemas.get_schemas_by_ids([])) + assert len(fresh) == 1, "the quoted column is skipped by a fresh schema" + + shared = graph_schemas.get_catalog_schemas() + assert [parsed_columns(shared) for _ in range(3)] == [fresh] * 3 + + +def test_forgetting_nodes_leaves_a_one_off_map_alone(catalog: _Catalog) -> None: + one_off = graph_schemas.get_schemas_by_ids([]) + node = one_off["shop"].get_table_node("orders") + + graph_schemas.forget_parsed_nodes(one_off) + + assert one_off["shop"].get_table_node("orders") is node + + +def test_column_index_follows_a_replaced_frame(catalog: _Catalog) -> None: + """The per-table index is tied to the frame it was read from.""" + schema = graph_schemas.get_schemas_by_ids([])["shop"] + assert schema.column_names_by_table()["customers"] == ("customer_id", "name") + + schema.columns_df = schema.columns_df[ + schema.columns_df["column_name"] != "name" + ].copy() + + assert schema.column_names_by_table()["customers"] == ("customer_id",) diff --git a/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py b/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py index ce78bd7df..5c3932d78 100644 --- a/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py +++ b/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py @@ -41,8 +41,9 @@ ) from auto_ontology.retrieval.data_access.custom_analyses import get_custom_analyses_ids from auto_ontology.retrieval.data_access.graph_schemas import ( - fetch_all_schema_ids, - get_schemas_by_ids, + catalog_schemas_lock, + forget_parsed_nodes, + get_catalog_schemas, ) from auto_ontology.retrieval.text_to_sql.formatters_util import ( format_semantic_context, @@ -796,8 +797,9 @@ def execute(self, state: AgentState) -> Dict[str, Any]: connector = resolve_connector_from_tables( path_state.get("relevant_tables"), connectors ) - schemas_ids = fetch_all_schema_ids() - schemas = get_schemas_by_ids(schemas_ids) + # The whole catalog, shared across attempts and questions and rebuilt + # only when the catalog itself changes -- see get_catalog_schemas. + schemas = get_catalog_schemas() degenerate_dialect = getattr(connector, "dialect", None) # Every constructed/reconstructed SQL passes through this node before @@ -860,9 +862,14 @@ def execute(self, state: AgentState) -> Dict[str, Any]: "path_state": path_state, } - validation_result = self._sql_parse_validation( - schemas, response.sql_code, dialects - ) + # The schema map is shared process-wide and builds nodes as it is + # parsed against, so only one parse touches it at a time, and each + # starts from a map no earlier parse has left its nodes on. + with catalog_schemas_lock: + forget_parsed_nodes(schemas) + validation_result = self._sql_parse_validation( + schemas, response.sql_code, dialects + ) if validation_result.get("error"): error_msg = validation_result["error"] diff --git a/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py b/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py index 3e15cb2b3..5d60d3720 100644 --- a/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py +++ b/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py @@ -296,15 +296,10 @@ def test_non_short_answer_validation_omits_strict_projection_check( "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.invoke_with_structured_output" ) @patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_schemas_by_ids", + "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_catalog_schemas", return_value={}, ) -@patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.fetch_all_schema_ids", - return_value=[], -) def test_parse_failure_does_not_call_intent_llm( - _mock_schema_ids: MagicMock, _mock_schemas: MagicMock, mock_invoke: MagicMock, ) -> None: @@ -372,15 +367,10 @@ def test_parse_result_without_catalog_table_is_invalid(_mock_parse: MagicMock) - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.invoke_with_structured_output" ) @patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_schemas_by_ids", + "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_catalog_schemas", return_value={}, ) -@patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.fetch_all_schema_ids", - return_value=[], -) def test_retry_threshold_still_runs_intent_llm( - _mock_schema_ids: MagicMock, _mock_schemas: MagicMock, mock_invoke: MagicMock, ) -> None: @@ -420,15 +410,10 @@ def test_retry_threshold_still_runs_intent_llm( "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.invoke_with_structured_output" ) @patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_schemas_by_ids", + "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_catalog_schemas", return_value={}, ) -@patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.fetch_all_schema_ids", - return_value=[], -) def test_final_full_pipeline_attempt_still_runs_intent_llm( - _mock_schema_ids: MagicMock, _mock_schemas: MagicMock, mock_invoke: MagicMock, ) -> None: diff --git a/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py b/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py index 6dc3db7fc..4a35c3658 100644 --- a/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py +++ b/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py @@ -428,8 +428,7 @@ def _validate( ) connector = SimpleNamespace(dialect=dialect) - monkeypatch.setattr(f"{_VALIDATION}.fetch_all_schema_ids", lambda: []) - monkeypatch.setattr(f"{_VALIDATION}.get_schemas_by_ids", lambda ids: {}) + monkeypatch.setattr(f"{_VALIDATION}.get_catalog_schemas", lambda: {}) monkeypatch.setattr( f"{_VALIDATION}.resolve_connector_from_tables", lambda tables, conns: connector ) From 6838a7b37134651d398c936742d9361f59582228 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:31:22 -0600 Subject: [PATCH 02/10] Reuse LLM connections and honour the configured timeout ChatNVIDIA asks for a session before every request and was handed a new one each time, so every model call paid a TCP and TLS handshake: 100 ms against integrate.api.nvidia.com, 250 ms against inference-api. Give each thread one session per client. Idle connections are dropped after five seconds, a request that fails immediately on a reused connection is sent once more on a new one, and no cookie is stored. The client also passed its own 60 second default on every request, which overrode LLM_INVOKE_TIMEOUT_S. Pass the configured value, so a generation that needs longer than a minute is no longer abandoned and retried. safe_invoke_structured_nr and safe_invoke_text_nr built a client per call. They now reuse one per thread. --- auto_ontology/utils/llm_invoke.py | 149 ++++++++++- auto_ontology/utils/tests/test_llm_invoke.py | 265 +++++++++++++++++++ 2 files changed, 405 insertions(+), 9 deletions(-) diff --git a/auto_ontology/utils/llm_invoke.py b/auto_ontology/utils/llm_invoke.py index bc1694f21..ada9ab657 100644 --- a/auto_ontology/utils/llm_invoke.py +++ b/auto_ontology/utils/llm_invoke.py @@ -11,7 +11,8 @@ import threading import time import weakref -from typing import Iterator, Type, TypeVar +from http.cookiejar import DefaultCookiePolicy +from typing import Any, Iterator, Type, TypeVar import requests as _requests from langchain_core.language_models import BaseChatModel @@ -183,16 +184,110 @@ def _is_retryable(e: Exception) -> bool: ) +# How long a connection may sit unused in a session before it is dropped rather +# than reused. A server or load balancer closes idle connections on its own +# schedule, and a request written to one it has just closed fails instead of +# being answered. Five seconds is the window httpx uses, so it is the one the +# ChatOpenAI path has always run with; it comfortably covers the gap between +# consecutive model calls in one question, which is what reuse is for. +LLM_KEEPALIVE_IDLE_S = float(os.environ.get("LLM_KEEPALIVE_IDLE_S", "5")) + +# A request written to a connection the far end already closed fails within a +# network round trip. A failure later than this is not that: the request was +# delivered and something went wrong while it was being answered. +_STALE_CONNECTION_WINDOW_S = 1.0 + + class _TimeoutSession(_requests.Session): - """requests.Session that enforces a default timeout on every request.""" + """requests.Session with a default timeout, safe to keep and reuse. + + ``ChatNVIDIA`` asks for a session before every request. Handing it a new + one each time means a TCP and TLS handshake per model call -- measured at + 100 ms against integrate.api.nvidia.com and 250 ms against + inference-api.nvidia.com -- which a question pays seven to twenty times. + Reusing one keeps the connection open between calls. + + Reuse brings one failure a fresh session cannot have: the far end closes a + connection just as a request is written to it. Three things keep that from + reaching the caller, and keep everything else as it was: + + * Connections idle for longer than ``max_idle`` are closed before the next + request, so one the far end has probably dropped is not written to. + * A request on a reused session that fails to connect *immediately* is + sent once more on a new connection. Immediately is the whole condition: + a connection that was already closed fails within a round trip, whereas + a failure seconds in means the endpoint received the request and broke + off while answering -- the failure a fresh session would have had too, + which is left to the caller's retry policy exactly as before. Timeouts + are never resent, whichever exception they arrive as. + * No cookie is ever stored. A one-shot session never sent any, and one set + by a load balancer would otherwise pin every later call to one backend. + """ - def __init__(self, timeout: float = LLM_INVOKE_TIMEOUT_S, **kwargs): + def __init__( + self, + timeout: float = LLM_INVOKE_TIMEOUT_S, + *, + max_idle: float = LLM_KEEPALIVE_IDLE_S, + **kwargs: Any, + ) -> None: super().__init__(**kwargs) self._default_timeout = timeout - - def request(self, method, url, **kwargs): + self._max_idle = max_idle + self._clock = time.monotonic + # When the last request finished, or None while no pooled connection + # can exist (never used, or the pool was just emptied). + self._last_used: float | None = None + self.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) + + def request(self, method: Any, url: Any, **kwargs: Any) -> _requests.Response: kwargs.setdefault("timeout", self._default_timeout) - return super().request(method, url, **kwargs) + started = self._clock() + reused = self._last_used is not None + if reused and started - self._last_used > self._max_idle: + self.close() + reused = False + self._last_used = None + try: + response = super().request(method, url, **kwargs) + except _requests.exceptions.ConnectionError as exc: + stale = ( + reused + and not isinstance(exc, _requests.exceptions.Timeout) + and self._clock() - started <= _STALE_CONNECTION_WINDOW_S + ) + if not stale: + raise + logger.info( + "LLM connection was closed while idle (%s); retrying once on a " + "new connection", + type(exc).__name__, + ) + self.close() + response = super().request(method, url, **kwargs) + self._last_used = self._clock() + return response + + +class _ThreadSessions: + """One reusable :class:`_TimeoutSession` per thread, for one client. + + A ``requests.Session`` is not documented as safe to share between threads, + and nothing here needs it to be: a question's model calls run one after + another on a single thread, so a session per thread reuses the connection + for exactly the calls that are close together, and concurrent questions + never touch each other's. + """ + + def __init__(self, timeout: float) -> None: + self._timeout = timeout + self._local = threading.local() + + def __call__(self) -> _TimeoutSession: + session = getattr(self._local, "session", None) + if session is None: + session = self._local.session = _TimeoutSession(self._timeout) + return session T = TypeVar("T", bound=BaseModel) @@ -249,8 +344,14 @@ def _build_client( base_url=base_url, temperature=temperature, max_tokens=max_tokens, + # Without this the client passes its own default of 60 seconds on + # every request, which silently overrides the session default below: + # LLM_INVOKE_TIMEOUT_S was never the limit in force on this path, and a + # generation that needed longer than a minute was abandoned, retried + # twice the same way, and reported as no answer. + timeout=LLM_INVOKE_TIMEOUT_S, ) - client._client.get_session_fn = lambda: _TimeoutSession(LLM_INVOKE_TIMEOUT_S) + client._client.get_session_fn = _ThreadSessions(LLM_INVOKE_TIMEOUT_S) return client @@ -319,6 +420,10 @@ def get_non_reasoning_llm_client( _NON_REASONING_STRONG_REFS: dict[int, object] = {} _REASONING_FALLBACK_CLIENT: BaseChatModel | None = None _REASONING_FALLBACK_LOCK = threading.Lock() +# Per thread: the default non-reasoning client and the factory that built it. +# See ``_get_default_non_reasoning_client`` for why it is per thread and why +# the factory is remembered. +_DEFAULT_NON_REASONING = threading.local() # How many ``bound`` links to follow. Real ``.bind()`` chains are a level or two # deep; the cap stops the walk on objects whose ``bound`` never ends — a @@ -363,6 +468,32 @@ def _get_reasoning_fallback_client() -> BaseChatModel: return _REASONING_FALLBACK_CLIENT +def _get_default_non_reasoning_client() -> BaseChatModel: + """This thread's non-reasoning client with default settings, built once. + + ``safe_invoke_text_nr`` and ``safe_invoke_structured_nr`` used to build a + client per call. A client owns its connection pool, so each of those calls + opened a new connection, did one request on it, and threw it away. A + failure to build is not remembered: the next call tries again. + + One per thread rather than one for the process, because these calls never + shared a client before and ``ChatNVIDIA`` is not built to be shared: it + stores the outgoing request on the client object and reads it back to send + it, so two threads on one client can each send the other's prompt. The + calls that benefit from reuse are consecutive ones on the same thread. + + The client is kept together with the factory that produced it and is + rebuilt if ``get_non_reasoning_llm_client`` is no longer that function. + Several test modules replace the factory; without this, a client one of + them built would outlive the patch and be handed to an unrelated caller. + """ + factory = get_non_reasoning_llm_client + cached = getattr(_DEFAULT_NON_REASONING, "client", None) + if cached is None or cached[0] is not factory: + cached = _DEFAULT_NON_REASONING.client = (factory, factory()) + return cached[1] + + def _reasoning_fallback_client(llm: object) -> BaseChatModel | None: """The reasoning client when *llm* is non-reasoning, else ``None``. @@ -691,7 +822,7 @@ def safe_invoke_text_nr(prompt: str) -> str: that single call when the non-reasoning endpoint stays down. """ try: - client = get_non_reasoning_llm_client() + client = _get_default_non_reasoning_client() except Exception as exc: _log_non_reasoning_fallback(exc) client = _get_reasoning_fallback_client() @@ -708,7 +839,7 @@ def safe_invoke_structured_nr( model for that single call. Returns ``None`` when both models fail. """ try: - client = get_non_reasoning_llm_client() + client = _get_default_non_reasoning_client() except Exception as exc: schema_name = getattr(schema, "__name__", str(schema)) _log_non_reasoning_fallback(exc, schema_name) diff --git a/auto_ontology/utils/tests/test_llm_invoke.py b/auto_ontology/utils/tests/test_llm_invoke.py index 3ce7558ec..a6c13ff3c 100644 --- a/auto_ontology/utils/tests/test_llm_invoke.py +++ b/auto_ontology/utils/tests/test_llm_invoke.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock import pytest +import requests from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage from pydantic import BaseModel @@ -27,6 +28,7 @@ def _isolate_fallback_state(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(llm_invoke, "_NON_REASONING_IDS", set()) monkeypatch.setattr(llm_invoke, "_NON_REASONING_STRONG_REFS", {}) monkeypatch.setattr(llm_invoke, "_REASONING_FALLBACK_CLIENT", None) + monkeypatch.setattr(llm_invoke, "_DEFAULT_NON_REASONING", threading.local()) class _Model: @@ -497,3 +499,266 @@ def unavailable(**_kwargs: object) -> object: monkeypatch.setattr(llm_invoke, "get_llm_client", unavailable) assert llm_invoke.invoke_with_structured_output(primary, [], _Schema) is None + + +# --------------------------------------------------------------------------- +# Connection reuse +# --------------------------------------------------------------------------- + + +class _Endpoint: + """A local HTTP/1.1 endpoint that records which connection served each request.""" + + def __init__(self) -> None: + import http.server + + endpoint = self + self.connections: list[tuple[str, int]] = [] + self.cookies: list[str | None] = [] + + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: # noqa: N802 — http.server's name + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + endpoint.connections.append(self.client_address) + endpoint.cookies.append(self.headers.get("Cookie")) + body = b"{}" + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Set-Cookie", "backend=node-7; Path=/") + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args: object) -> None: + pass + + self._server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self._server.server_address[1]}/v1/chat" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _Endpoint() + yield server + server.close() + + +def test_consecutive_calls_share_one_connection(endpoint: _Endpoint) -> None: + session = llm_invoke._TimeoutSession(5) + + for _ in range(3): + assert session.post(endpoint.url, json={"q": 1}).status_code == 200 + + assert len(set(endpoint.connections)) == 1 + + +def test_a_connection_left_idle_is_replaced_not_reused(endpoint: _Endpoint) -> None: + session = llm_invoke._TimeoutSession(5, max_idle=0.0) + + session.post(endpoint.url, json={}) + session.post(endpoint.url, json={}) + + assert len(set(endpoint.connections)) == 2 + + +def test_reused_session_sends_no_cookie(endpoint: _Endpoint) -> None: + """A one-shot session never carried one; a kept one must not start to.""" + session = llm_invoke._TimeoutSession(5) + + session.post(endpoint.url, json={}) + session.post(endpoint.url, json={}) + + assert endpoint.cookies == [None, None] + assert len(session.cookies) == 0 + + +class _FlakySession(llm_invoke._TimeoutSession): + """Raises the queued errors, one per send, before letting requests through.""" + + def __init__(self, errors: list[Exception], **kwargs: Any) -> None: + super().__init__(5, **kwargs) + self.errors = errors + self.sends = 0 + + def send(self, request: Any, **kwargs: Any) -> Any: + self.sends += 1 + if self.errors: + error = self.errors.pop(0) + # A callable is called at failure time, so it can move the clock. + raise error() if callable(error) else error + return super().send(request, **kwargs) + + +def test_stale_connection_is_retried_once_on_a_new_one(endpoint: _Endpoint) -> None: + session = _FlakySession([]) + session.post(endpoint.url, json={}) + session.errors.append(requests.exceptions.ConnectionError("Connection aborted.")) + + response = session.post(endpoint.url, json={}) + + assert response.status_code == 200 + assert session.sends == 3 + assert len(set(endpoint.connections)) == 2 + + +def test_a_failure_while_the_endpoint_was_answering_is_not_resent( + endpoint: _Endpoint, +) -> None: + """Seconds in, the request was delivered: resending would deliver it twice. + + This is the failure a fresh session has too, so it goes to the caller's + retry policy as it always did. requests reports a stalled response body as + a plain ConnectionError as well, which is why elapsed time decides this + and not the exception type. + """ + session = _FlakySession([]) + session.post(endpoint.url, json={}) + now = [session._clock()] + session._clock = lambda: now[0] + + def dropped_after_thirty_seconds() -> Exception: + now[0] += 30.0 + return requests.exceptions.ConnectionError("Read timed out.") + + session.errors.append(dropped_after_thirty_seconds) + + with pytest.raises(requests.exceptions.ConnectionError): + session.post(endpoint.url, json={}) + + assert session.sends == 2 + + +def test_first_request_connection_failure_is_not_retried(endpoint: _Endpoint) -> None: + """Nothing was pooled yet, so the failure is the endpoint's, not staleness.""" + session = _FlakySession([requests.exceptions.ConnectionError("refused")]) + + with pytest.raises(requests.exceptions.ConnectionError): + session.post(endpoint.url, json={}) + + assert session.sends == 1 + + +def test_timeout_on_a_reused_session_is_not_retried(endpoint: _Endpoint) -> None: + session = _FlakySession([]) + session.post(endpoint.url, json={}) + session.errors.append(requests.exceptions.ConnectTimeout("timed out")) + + with pytest.raises(requests.exceptions.ConnectTimeout): + session.post(endpoint.url, json={}) + + assert session.sends == 2 + + +def test_a_second_connection_failure_is_raised(endpoint: _Endpoint) -> None: + session = _FlakySession([]) + session.post(endpoint.url, json={}) + session.errors.extend( + [ + requests.exceptions.ConnectionError("aborted"), + requests.exceptions.ConnectionError("refused"), + ] + ) + + with pytest.raises(requests.exceptions.ConnectionError, match="refused"): + session.post(endpoint.url, json={}) + + # The failed retry left nothing pooled, so the next call starts clean. + assert session.post(endpoint.url, json={}).status_code == 200 + + +def test_each_thread_gets_its_own_session() -> None: + sessions = llm_invoke._ThreadSessions(5) + seen: list[object] = [] + + worker = threading.Thread(target=lambda: seen.append(sessions())) + worker.start() + worker.join() + + assert sessions() is sessions() + assert seen[0] is not sessions() + + +def test_nvidia_client_reuses_sessions_and_honours_the_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The client's own 60s default used to override the configured timeout.""" + monkeypatch.setattr(llm_invoke, "LLM_INVOKE_TIMEOUT_S", 137) + + client = llm_invoke._build_client( + model="nvidia/nemotron-3.5-lightning-30b-a3b", + api_key="nvapi-test", + base_url="http://127.0.0.1:9/v1", + temperature=0.0, + max_tokens=16, + ) + + assert client._client.timeout == 137 + assert client._client.get_session_fn() is client._client.get_session_fn() + + +def test_default_non_reasoning_client_is_built_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + built: list[object] = [] + + def factory(**_kwargs: object) -> object: + built.append(object()) + return built[-1] + + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", factory) + monkeypatch.setattr( + llm_invoke, + "invoke_with_structured_output", + lambda llm, _messages, _schema: llm, + ) + monkeypatch.setattr(llm_invoke, "safe_invoke_text", lambda llm, _prompt: llm) + + first = llm_invoke.safe_invoke_structured_nr([], _Schema) + second = llm_invoke.safe_invoke_structured_nr([], _Schema) + third = llm_invoke.safe_invoke_text_nr("question") + + assert first is second is third + assert len(built) == 1 + + +def test_default_non_reasoning_client_is_not_shared_between_threads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """These calls built a client each before; they must still not share one. + + ChatNVIDIA keeps the outgoing request on the client object between + building it and sending it, so a client used from two threads can send one + thread's prompt on behalf of the other. + """ + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", lambda: object()) + monkeypatch.setattr(llm_invoke, "safe_invoke_text", lambda llm, _prompt: llm) + other: list[object] = [] + + worker = threading.Thread( + target=lambda: other.append(llm_invoke.safe_invoke_text_nr("question")) + ) + worker.start() + worker.join() + + mine = llm_invoke.safe_invoke_text_nr("question") + assert mine is llm_invoke.safe_invoke_text_nr("question") + assert other[0] is not mine + + +def test_default_non_reasoning_client_follows_a_replaced_factory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(llm_invoke, "safe_invoke_text", lambda llm, _prompt: llm) + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", lambda: "first") + assert llm_invoke.safe_invoke_text_nr("question") == "first" + + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", lambda: "second") + assert llm_invoke.safe_invoke_text_nr("question") == "second" From a98e3c9d6087d6e3d387237d8197ff4c9db54575 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:31:22 -0600 Subject: [PATCH 03/10] Embed each query string once per process A question searches for the same entity during retrieval, preparation, hub-sibling ranking and reconstruction, and each search sent its own request to the embedding endpoint, at 0.7 to 1.1 s each. Keep the endpoint's answer per exact request and return it bit for bit. Concurrent requests for one string share a call. Failures and blank queries are never stored, and nothing is batched. --- auto_ontology/utils/retriever.py | 138 ++++++++++- .../utils/tests/test_retriever_query_cache.py | 227 ++++++++++++++++++ 2 files changed, 358 insertions(+), 7 deletions(-) create mode 100644 auto_ontology/utils/tests/test_retriever_query_cache.py diff --git a/auto_ontology/utils/retriever.py b/auto_ontology/utils/retriever.py index cc04b16bd..61b2b775b 100644 --- a/auto_ontology/utils/retriever.py +++ b/auto_ontology/utils/retriever.py @@ -7,6 +7,11 @@ from __future__ import annotations import logging +import os +import threading +from array import array +from collections import OrderedDict +from concurrent.futures import Future from typing import Any from auto_ontology.utils.embedding import EmbedParams, embed_texts, get_embed_params @@ -22,6 +27,99 @@ # endpoint costs three 60s attempts, not three times six. _QUERY_TIMEOUT_S = 60.0 +# Query vectors kept per process. One 2048-dimension vector is 16 KB as packed +# doubles, so the default holds about 16 MB. Zero turns the cache off. +_QUERY_CACHE_SIZE = int(os.environ.get("AUTO_ONTOLOGY_QUERY_EMBED_CACHE_SIZE", "1024")) + + +class _QueryEmbeddingCache: + """Embed each distinct query string once. + + One question embeds the same few strings again and again: each entity is + searched for attributes during retrieval, for tables during preparation, + again to rank hub siblings, and once more by every reconstruction that + goes looking for a missing table; a full-pipeline retry repeats all of it. + Each of those was its own request to the embedding endpoint, on the + question's critical path whenever nothing else was running, and each one a + chance to be rate-limited into a multi-second backoff. + + The cached value is the endpoint's own answer for that exact request -- + same model, same input type, same text -- held as packed doubles, so a hit + returns bit-for-bit the vector a new request would have. Nothing is + approximated and nothing is batched: a batch is a different request and is + not guaranteed the same floats. + + Concurrent requests for a string not yet cached share one call. Retrieval + searches the full question under two labels at the same moment, and without + this both would miss and both would embed it. A failure is never stored: + every waiter gets the exception, and the next request tries again. + """ + + def __init__(self, max_entries: int) -> None: + self._max_entries = max_entries + self._lock = threading.Lock() + self._vectors: OrderedDict[tuple, array] = OrderedDict() + self._pending: dict[tuple, Future] = {} + + def get(self, key: tuple, compute: Any) -> list[float] | None: + """The vector for *key*, calling *compute* only if nobody has it yet. + + ``None`` from *compute* (a blank query) is returned and not kept. + """ + if self._max_entries <= 0: + return compute() + + with self._lock: + packed = self._vectors.get(key) + if packed is not None: + self._vectors.move_to_end(key) + return packed.tolist() + pending = self._pending.get(key) + if pending is None: + pending = self._pending[key] = Future() + owner = True + else: + owner = False + + if not owner: + vector = pending.result() + return None if vector is None else list(vector) + + try: + vector = compute() + except BaseException as exc: + with self._lock: + self._pending.pop(key, None) + pending.set_exception(exc) + raise + + # Packing can only fail for something that is not a flat list of + # numbers. That is not this cache's to judge: the vector goes on to the + # store exactly as it would have, and is simply not kept. Whatever + # happens here, the waiters are released. + packed: array | None = None + if vector is not None: + try: + packed = array("d", vector) + except (TypeError, ValueError, OverflowError): + packed = None + + with self._lock: + self._pending.pop(key, None) + if packed is not None: + self._vectors[key] = packed + while len(self._vectors) > self._max_entries: + self._vectors.popitem(last=False) + pending.set_result(vector) + return vector + + def clear(self) -> None: + with self._lock: + self._vectors.clear() + + +_query_embeddings = _QueryEmbeddingCache(_QUERY_CACHE_SIZE) + class Retriever: """Embed a query and run a k-NN search against one :class:`PostgresVDB`. @@ -54,13 +152,7 @@ def query( embedded, with the endpoint's HTTP status in the message, so callers can tell a transient failure from an empty result. """ - vector = embed_texts( - [text], - self.embed_params, - "query", - timeout=_QUERY_TIMEOUT_S, - retries=0, - )[0] + vector = self._embed_query(text) if vector is None: raise ValueError("Cannot search for a blank query.") where = (vdb_kwargs or {}).get("where") @@ -74,6 +166,38 @@ def query( for hit in hits[:top_k] ] + def _embed_query(self, text: str) -> list[float] | None: + """The query vector for *text*, embedded at most once per process. + + The key is everything that decides the request ``embed_texts`` sends: + where it goes, which model answers, the modality, and the text as it + is sent (stripped). See :class:`_QueryEmbeddingCache`. + + The embedding function itself is part of the key as well. In the + running service it is always the same function; a test that swaps in + its own embedder gets its own entries instead of vectors another + embedder produced for the same string. + """ + params = self.embed_params + embed = embed_texts + key = ( + embed, + params.embed_invoke_url, + params.model_name, + params.embed_modality, + text.strip() if isinstance(text, str) else text, + ) + return _query_embeddings.get( + key, + lambda: embed( + [text], + params, + "query", + timeout=_QUERY_TIMEOUT_S, + retries=0, + )[0], + ) + _data_retriever: Retriever | None = None _semantic_retriever: Retriever | None = None diff --git a/auto_ontology/utils/tests/test_retriever_query_cache.py b/auto_ontology/utils/tests/test_retriever_query_cache.py new file mode 100644 index 000000000..0deb4af59 --- /dev/null +++ b/auto_ontology/utils/tests/test_retriever_query_cache.py @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A query string is embedded once, and what comes back is exactly that vector. + +One question searches for the same entity several times across retrieval, +preparation and reconstruction. Each search used to be its own request to the +embedding endpoint. Reusing the answer is only acceptable if it is the same +answer, and if a failed or blank request is never mistaken for one. +""" + +from __future__ import annotations + +import threading +from typing import Any + +import pytest + +from auto_ontology.utils import retriever as retriever_mod +from auto_ontology.utils.embedding import EmbedParams +from auto_ontology.utils.retriever import Retriever, _QueryEmbeddingCache + + +class _Store: + """Records the vectors it is searched with.""" + + def __init__(self) -> None: + self.searched: list[list[float]] = [] + + def retrieval(self, queries: list, top_k: int = 10, **kwargs: Any) -> list: + self.searched.extend(queries) + return [[] for _ in queries] + + +class _Embedder: + """Stands in for ``embed_texts``, counting requests per text.""" + + def __init__(self) -> None: + self.requests: list[str] = [] + self.fail_with: Exception | None = None + self.gate: threading.Event | None = None + + def __call__( + self, texts: list[str], params: Any, input_type: str, **kw: Any + ) -> list: + self.requests.extend(texts) + if self.gate is not None: + assert self.gate.wait(5) + if self.fail_with is not None: + raise self.fail_with + if not texts[0].strip(): + return [None] + # Values that do not survive float32, so a lossy cache would show. + return [[len(texts[0]) / 3.0, 0.1 + 0.2, 1e-17]] + + +@pytest.fixture +def embedder(monkeypatch: pytest.MonkeyPatch) -> _Embedder: + embedder = _Embedder() + monkeypatch.setattr(retriever_mod, "embed_texts", embedder) + monkeypatch.setattr(retriever_mod, "_query_embeddings", _QueryEmbeddingCache(8)) + return embedder + + +def _retriever(model: str = "model-a") -> tuple[Retriever, _Store]: + store = _Store() + return Retriever(store, EmbedParams(model, "http://embed.test/v1")), store # type: ignore[arg-type] + + +def test_repeated_query_is_embedded_once(embedder: _Embedder) -> None: + retriever, store = _retriever() + + retriever.query("total revenue") + retriever.query("total revenue", top_k=3) + retriever.query(" total revenue ") + + assert embedder.requests == ["total revenue"] + assert store.searched[0] == store.searched[1] == store.searched[2] + + +def test_cached_vector_is_bit_identical(embedder: _Embedder) -> None: + retriever, store = _retriever() + + retriever.query("margin") + retriever.query("margin") + + first, second = store.searched + assert first == [len("margin") / 3.0, 0.1 + 0.2, 1e-17] + assert [value.hex() for value in second] == [value.hex() for value in first] + assert isinstance(second, list) + + +def test_different_text_or_model_is_a_different_request(embedder: _Embedder) -> None: + retriever_a, _ = _retriever("model-a") + retriever_b, _ = _retriever("model-b") + + retriever_a.query("orders") + retriever_a.query("Orders") + retriever_b.query("orders") + + assert embedder.requests == ["orders", "Orders", "orders"] + + +def test_failure_is_not_cached(embedder: _Embedder) -> None: + retriever, _ = _retriever() + embedder.fail_with = RuntimeError("502 Bad Gateway") + + with pytest.raises(RuntimeError, match="502"): + retriever.query("orders") + + embedder.fail_with = None + retriever.query("orders") + + assert embedder.requests == ["orders", "orders"] + + +def test_blank_query_still_raises_and_is_not_cached(embedder: _Embedder) -> None: + retriever, _ = _retriever() + + for _ in range(2): + with pytest.raises(ValueError, match="blank query"): + retriever.query(" ") + + assert embedder.requests == [" ", " "] + + +def test_concurrent_requests_for_one_string_share_a_call(embedder: _Embedder) -> None: + """Retrieval searches the question under two labels at the same moment.""" + retriever, store = _retriever() + embedder.gate = threading.Event() + threads = [ + threading.Thread(target=retriever.query, args=("how many orders",)) + for _ in range(4) + ] + for thread in threads: + thread.start() + while not embedder.requests: + pass + embedder.gate.set() + for thread in threads: + thread.join(5) + + assert embedder.requests == ["how many orders"] + assert len(store.searched) == 4 + assert all(vector == store.searched[0] for vector in store.searched) + + +def test_concurrent_waiters_see_the_owners_failure(embedder: _Embedder) -> None: + retriever, _ = _retriever() + embedder.gate = threading.Event() + embedder.fail_with = RuntimeError("503 Service Unavailable") + errors: list[BaseException] = [] + + def search() -> None: + try: + retriever.query("orders") + except BaseException as exc: # noqa: BLE001 — recorded for the assertion + errors.append(exc) + + threads = [threading.Thread(target=search) for _ in range(3)] + for thread in threads: + thread.start() + while not embedder.requests: + pass + embedder.gate.set() + for thread in threads: + thread.join(5) + + assert len(errors) == 3 + assert all("503" in str(error) for error in errors) + + +def test_a_vector_that_cannot_be_packed_is_passed_on_and_strands_nobody() -> None: + """Whatever the endpoint returned still reaches the store, and every waiter.""" + cache = _QueryEmbeddingCache(8) + gate = threading.Event() + odd = [[0.1, 0.2]] # nested: not a flat list of numbers + calls: list[int] = [] + + def compute() -> list: + calls.append(1) + assert gate.wait(5) + return odd + + results: list[object] = [] + threads = [ + threading.Thread(target=lambda: results.append(cache.get(("k",), compute))) + for _ in range(3) + ] + for thread in threads: + thread.start() + while not calls: + pass + gate.set() + for thread in threads: + thread.join(5) + + assert not any(thread.is_alive() for thread in threads) + assert results == [odd, odd, odd] + # Not kept, so the next request asks again instead of reusing it. + assert cache.get(("k",), compute) == odd + assert len(calls) == 2 + + +def test_oldest_entry_is_evicted_first(embedder: _Embedder) -> None: + retriever, _ = _retriever() + retriever_mod._query_embeddings = _QueryEmbeddingCache(2) + + retriever.query("a") + retriever.query("b") + retriever.query("a") # a is now the most recently used + retriever.query("c") # evicts b + retriever.query("a") + retriever.query("b") + + assert embedder.requests == ["a", "b", "c", "b"] + + +def test_size_zero_disables_the_cache(embedder: _Embedder) -> None: + retriever, _ = _retriever() + retriever_mod._query_embeddings = _QueryEmbeddingCache(0) + + retriever.query("orders") + retriever.query("orders") + + assert embedder.requests == ["orders", "orders"] From 7fd1ffc1a73aab3927f42547fb0230f4853291a9 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:31:22 -0600 Subject: [PATCH 04/10] Warm the chat worker before its first question A worker left the catalog store's pool, both vector stores and the catalog schema map to be opened by the first question it served, and a worker is replaced on every cancel. Open them once the worker has reported ready. Warming is read-only and best-effort: it never creates a collection, and it stops if the catalog store cannot be reached. --- .../server/chat/tests/test_worker.py | 94 ++++++++++++++++++- auto_ontology/server/chat/worker.py | 50 ++++++++++ auto_ontology/vdb/postgres.py | 16 ++++ auto_ontology/vdb/tests/test_records.py | 14 +++ 4 files changed, 173 insertions(+), 1 deletion(-) diff --git a/auto_ontology/server/chat/tests/test_worker.py b/auto_ontology/server/chat/tests/test_worker.py index b283cf77a..85c5948cc 100644 --- a/auto_ontology/server/chat/tests/test_worker.py +++ b/auto_ontology/server/chat/tests/test_worker.py @@ -33,6 +33,8 @@ def _run_loop( connector_batches: list[list[str]], *, evidence: str | None = None, + out_q: _FakeQueue | None = None, + prewarm: Any = lambda *retrievers: None, ) -> dict: """Drive one ASK through ``_worker_loop`` and return the agent payload it built. @@ -79,9 +81,11 @@ def fake_stream(payload: dict) -> Any: monkeypatch.setattr( "auto_ontology.retrieval.text_to_sql.main.stream_agent_response", fake_stream ) + # Warming opens real connections; these tests are about the request loop. + monkeypatch.setattr(worker_module, "_prewarm", prewarm) in_q = _FakeQueue([(worker_module._MSG_ASK, ("q", None, None, None, [], evidence))]) - worker_module._worker_loop(in_q, _FakeQueue()) + worker_module._worker_loop(in_q, out_q or _FakeQueue()) return captured @@ -122,3 +126,91 @@ def test_worker_forwards_evidence_without_changing_question( assert payload["question"] == "q" assert payload["processing_question"] == "q" assert payload["evidence"] == "status means accounts.status" + + +class _Vdb: + def __init__(self, error: Exception | None = None) -> None: + self.error = error + self.warmed = 0 + + def warm(self) -> bool: + self.warmed += 1 + if self.error is not None: + raise self.error + return True + + +class _Retriever: + def __init__(self, vdb: _Vdb) -> None: + self.vdb = vdb + + +def _stub_prewarm_targets( + monkeypatch: MonkeyPatch, *, store_error: Exception | None = None +) -> list[str]: + calls: list[str] = [] + + class _Store: + def query_read(self, statement: str) -> list: + calls.append(statement) + if store_error is not None: + raise store_error + return [] + + def empty_catalog() -> dict: + calls.append("schema map") + raise KeyError("database_name") + + monkeypatch.setattr("auto_ontology.dal.session.store", lambda: _Store()) + monkeypatch.setattr( + "auto_ontology.retrieval.data_access.graph_schemas.get_catalog_schemas", + empty_catalog, + ) + return calls + + +def test_prewarm_opens_every_store_and_survives_a_failing_one( + monkeypatch: MonkeyPatch, +) -> None: + """One store being down must not leave the others cold, or stop the worker.""" + calls = _stub_prewarm_targets(monkeypatch) + broken = _Vdb(RuntimeError("connection refused")) + healthy = _Vdb() + + worker_module._prewarm(_Retriever(broken), _Retriever(healthy)) + + assert calls == ["SELECT 1", "schema map"] + assert (broken.warmed, healthy.warmed) == (1, 1) + + +def test_prewarm_stops_when_the_database_is_unreachable( + monkeypatch: MonkeyPatch, +) -> None: + """The vector stores share that database and connect with no timeout.""" + calls = _stub_prewarm_targets( + monkeypatch, store_error=RuntimeError("connection timeout expired") + ) + vdb = _Vdb() + + worker_module._prewarm(_Retriever(vdb)) + + assert calls == ["SELECT 1"] + assert vdb.warmed == 0 + + +def test_worker_reports_ready_before_it_warms(monkeypatch: MonkeyPatch) -> None: + """The pool waits on READY with no liveness check; warming must not precede it.""" + out_q = _FakeQueue() + ready_when_warming: list[bool] = [] + + def record(*_retrievers: object) -> None: + ready_when_warming.append((worker_module._TAG_READY, None) in out_q.puts) + + _run_loop( + monkeypatch, + connector_batches=[["boot-connector"]], + out_q=out_q, + prewarm=record, + ) + + assert ready_when_warming == [True] diff --git a/auto_ontology/server/chat/worker.py b/auto_ontology/server/chat/worker.py index 2dcb1198b..ce3c61a4a 100644 --- a/auto_ontology/server/chat/worker.py +++ b/auto_ontology/server/chat/worker.py @@ -56,6 +56,49 @@ _TERMINATE_GRACE_S = 1.0 +def _prewarm(*retrievers: Any) -> None: + """Pay the first question's one-time costs before there is a question. + + Importing the agent and building the singletons leaves several things + unopened: the catalog store's connection pool, each vector store's engine + and table binding, and the catalog schema map SQL validation parses + against. Each is created on first use, so without this the first question + a worker serves is slower than every later one -- and a worker is replaced + on every cancel, so "first" recurs. + + Best-effort and read-only. A step that fails is skipped and happens lazily + at its usual place instead. An empty catalog, for one, has no schema map + to build yet. + + The catalog store goes first and decides whether anything else is tried. + It is the one connection here with a connect timeout, and the vector + stores live in the same database: if it cannot be reached there is nothing + to warm, and trying would only wait on a connection that is not coming. + """ + from auto_ontology.dal.session import store + from auto_ontology.retrieval.data_access.graph_schemas import get_catalog_schemas + + try: + store().query_read("SELECT 1") + except Exception as exc: # noqa: BLE001 — warming is optional + logger.info( + "Prewarm skipped: the catalog store is not reachable (%s: %s)", + type(exc).__name__, + exc, + ) + return + + steps: list[tuple[str, Any]] = [("catalog schema map", get_catalog_schemas)] + for retriever in retrievers: + steps.append(("vector store", lambda r=retriever: r.vdb.warm())) + + for name, step in steps: + try: + step() + except Exception as exc: # noqa: BLE001 — warming is optional + logger.info("Prewarm skipped the %s: %s: %s", name, type(exc).__name__, exc) + + def _worker_loop( in_q: "mp.Queue[tuple[str, Any]]", out_q: "mp.Queue[tuple[str, Any]]", @@ -123,6 +166,13 @@ def _worker_loop( except Exception: # noqa: BLE001 return + # After READY, not before. The pool waits for READY with no timeout and no + # liveness check, so a worker that died while warming -- building the + # schema map for a very large catalog is the heaviest thing it does at + # start -- would leave that wait blocked for good. Once READY is out, a + # death here is seen the way a death mid-question is: by events(). + _prewarm(data_retriever, semantic_retriever) + while True: try: message = in_q.get() diff --git a/auto_ontology/vdb/postgres.py b/auto_ontology/vdb/postgres.py index 68f48c083..2733a7dce 100644 --- a/auto_ontology/vdb/postgres.py +++ b/auto_ontology/vdb/postgres.py @@ -339,6 +339,22 @@ def create_index(self, **kwargs: Any) -> str: self._get_store() return self.collection_name + def warm(self) -> bool: + """Open the connection pool and bind the store ahead of the first search. + + Otherwise the first search a process runs pays for the engine, a pooled + connection and the store's table inspection before it can query. + + Read-only, unlike :meth:`create_index`: a collection that does not + exist yet is left that way, and the first write creates it as before. + Returns whether the store is now ready. + """ + if self._store is not None: + return True + if not self._table_exists(): + return False + return self._get_store() is not None + # ------------------------------------------------------------------ # Read / write # ------------------------------------------------------------------ diff --git a/auto_ontology/vdb/tests/test_records.py b/auto_ontology/vdb/tests/test_records.py index 310ee43c0..4db83b5db 100644 --- a/auto_ontology/vdb/tests/test_records.py +++ b/auto_ontology/vdb/tests/test_records.py @@ -171,3 +171,17 @@ def _embed(texts: list[str], params: Any, input_type: str, **kw: Any) -> list: assert ( retriever.query("who buys", vdb_kwargs={"where": {"database_name": "x"}}) == [] ) + + +def test_warming_never_creates_a_collection(vdb: Any) -> None: + """Warming runs at worker start, before anything may have been ingested.""" + assert vdb.warm() is False + assert vdb._table_exists() is False + + write_embedded_rows(vdb, [_row("orders", _unit(0))]) + vdb.close() + + assert vdb.warm() is True + assert vdb.warm() is True + [hits] = vdb.retrieval([_unit(0)], top_k=1) + assert hits[0]["metadata"]["id"] == "orders" From ccd5d6ffc0806dfd4f404d7fa72a496288090161 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:31:22 -0600 Subject: [PATCH 05/10] Log where a text-to-SQL run spent its time One line per run, slowest node first, with visit counts. Logging only. --- auto_ontology/retrieval/text_to_sql/main.py | 49 +++++++++++ .../text_to_sql/tests/test_node_timings.py | 87 +++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py diff --git a/auto_ontology/retrieval/text_to_sql/main.py b/auto_ontology/retrieval/text_to_sql/main.py index 6fd147cf4..58eb81ab8 100644 --- a/auto_ontology/retrieval/text_to_sql/main.py +++ b/auto_ontology/retrieval/text_to_sql/main.py @@ -135,6 +135,47 @@ def __init__( ) +class _NodeTimings: + """Where one run's wall time went, node by node. + + Nodes run strictly one after another, so the time between two consecutive + node completions belongs to the later node. That needs nothing from the + nodes themselves and holds for both ways the graph is streamed. + + The answer to "why was that question slow" is otherwise only recoverable + by lining up log timestamps by hand, and it differs per question: one + spends its time generating SQL, another in four rounds of reconstruction, + another waiting on the database. + """ + + def __init__(self) -> None: + self._started = time.perf_counter() + self._last = self._started + self._seconds: dict[str, float] = {} + self._visits: dict[str, int] = {} + + def node_finished(self, node_name: str) -> None: + now = time.perf_counter() + if node_name not in _TRANSPARENT_NODES: + self._seconds[node_name] = ( + self._seconds.get(node_name, 0.0) + now - self._last + ) + self._visits[node_name] = self._visits.get(node_name, 0) + 1 + self._last = now + + def summary(self) -> str: + """Slowest node first, e.g. ``total 41.3s | reconstruct_sql 19.8s (x3) | ...``.""" + parts = [f"total {time.perf_counter() - self._started:.1f}s"] + for node_name, seconds in sorted( + self._seconds.items(), key=lambda item: item[1], reverse=True + ): + visits = self._visits[node_name] + parts.append( + f"{node_name} {seconds:.1f}s" + (f" (x{visits})" if visits > 1 else "") + ) + return " | ".join(parts) + + def _build_state(payload: TextToSQLPayload) -> AgentState: custom_prompts = payload.get("custom_prompts", "") acronyms = payload.get("acronyms", []) @@ -441,6 +482,7 @@ def stream_agent_response( streamed_sql: str | None = None last_node: str | None = None + timings = _NodeTimings() try: while True: final_state = dict(state) @@ -471,6 +513,7 @@ def stream_agent_response( logger.info("--- AGENT STEP ---") for node_name, node_output in chunk.items(): + timings.node_finished(node_name) if node_name in _TRANSPARENT_NODES: _merge_node_output(final_state, node_output) continue @@ -532,10 +575,12 @@ def stream_agent_response( answer["thoughts"] = thoughts_summary elapsed = time.perf_counter() - t0 logger.debug("Final answer (%.2fs):\n%s", elapsed, answer) + logger.info("Text-to-SQL timing: %s", timings.summary()) yield {"type": "result", "answer": answer} except Exception as exc: logger.exception("Error during agent stream") + logger.info("Text-to-SQL timing (failed run): %s", timings.summary()) yield { "type": "error", "message": f"Agent failed after {last_node or 'graph_start'}: {exc}", @@ -564,6 +609,7 @@ def get_agent_response_with_state(payload: TextToSQLPayload) -> dict: # Required by auto_ontology/retrieval/interactive/coordinator.py to persist path_state across turns/phases. state = _build_state(payload) final_state = dict(state) + timings = _NodeTimings() try: while True: @@ -571,6 +617,7 @@ def get_agent_response_with_state(payload: TextToSQLPayload) -> dict: terminal_node: str | None = None for step in app.stream(state, config={"recursion_limit": 45}): for node_name, node_output in step.items(): + timings.node_finished(node_name) if node_name not in _TRANSPARENT_NODES: terminal_node = node_name _merge_node_output(final_state, node_output) @@ -583,8 +630,10 @@ def get_agent_response_with_state(payload: TextToSQLPayload) -> dict: ) state = _restore_for_full_pipeline_retry(final_state) _apply_last_deterministic_sql_fallback(final_state, terminal_node) + logger.info("Text-to-SQL timing: %s", timings.summary()) except Exception as exc: logger.exception("Error during agent stream in get_agent_response_with_state") + logger.info("Text-to-SQL timing (failed run): %s", timings.summary()) # The stream may have already produced a valid, executed SQL query # (e.g. several reconstruction rounds succeeded) before a later node # raised — most commonly GraphRecursionError from an intent-validation diff --git a/auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py b/auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py new file mode 100644 index 000000000..e238594d6 --- /dev/null +++ b/auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A run reports where its time went, so a slow question can be explained.""" + +from __future__ import annotations + +import importlib +import logging +from types import ModuleType +from typing import Any + +import pytest + +from auto_ontology.utils import llm_invoke + + +@pytest.fixture +def main_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + monkeypatch.setattr(llm_invoke, "get_llm_client", lambda **_kwargs: None) + monkeypatch.setattr( + llm_invoke, "get_non_reasoning_llm_client", lambda **_kwargs: None + ) + return importlib.import_module("auto_ontology.retrieval.text_to_sql.main") + + +def _clock(monkeypatch: pytest.MonkeyPatch, main_module: ModuleType) -> list[float]: + """Replace the timer's clock with one the test advances by hand.""" + now = [100.0] + monkeypatch.setattr(main_module.time, "perf_counter", lambda: now[0]) + return now + + +def test_time_between_completions_belongs_to_the_later_node( + main_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + now = _clock(monkeypatch, main_module) + timings = main_module._NodeTimings() + + for node_name, seconds in [ + ("_entry_router", 0.25), + ("question_intent", 2.0), + ("construct_sql_from_candidates", 18.0), + ("validate_sql_query", 3.0), + ("reconstruct_sql", 9.0), + ("validate_sql_query", 4.5), + ]: + now[0] += seconds + timings.node_finished(node_name) + + assert timings.summary() == ( + "total 36.8s | construct_sql_from_candidates 18.0s | reconstruct_sql 9.0s" + " | validate_sql_query 7.5s (x2) | question_intent 2.0s" + ) + + +def test_a_streamed_run_logs_its_timing( + main_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + now = _clock(monkeypatch, main_module) + + class _App: + def stream(self, state: dict, **_kwargs: Any) -> Any: + for node_name, seconds in [ + ("question_intent", 1.5), + ("format_and_respond", 0.5), + ]: + now[0] += seconds + yield ( + "updates", + {node_name: {"path_state": {"final_response": {"response": "ok"}}}}, + ) + + monkeypatch.setattr(main_module, "app", _App()) + monkeypatch.setattr(main_module, "_build_state", lambda payload: {"path_state": {}}) + + with caplog.at_level(logging.INFO, logger=main_module.__name__): + events = list(main_module.stream_agent_response({"question": "q"})) + + assert events[-1] == {"type": "result", "answer": {"response": "ok"}} + assert ( + "Text-to-SQL timing: total 2.0s | question_intent 1.5s | format_and_respond 0.5s" + in caplog.text + ) From 8675656c7224f723425b56b70a2985d573eeee8a Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:31:22 -0600 Subject: [PATCH 06/10] Document the keep-alive and query embedding cache settings --- .env.example | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.env.example b/.env.example index 3b9458913..8ffa066b3 100644 --- a/.env.example +++ b/.env.example @@ -172,6 +172,15 @@ AUTH_SECRET=myverylongsecret123 # Max concurrent LLM requests across all worker threads, to stay under the # hosted endpoint's per-worker request cap (exceeding it surfaces as HTTP 503). # LLM_MAX_INFLIGHT=6 +# How long, in seconds, a connection to the LLM endpoint may sit unused before +# it is replaced rather than reused for the next call. Applies to models served +# through ChatNVIDIA. Lower it if the endpoint closes idle connections sooner. +# LLM_KEEPALIVE_IDLE_S=5 + +# --- Retrieval tuning (auto_ontology/utils/retriever.py) --- +# How many distinct query strings keep their embedding in memory, so a string +# searched for more than once is embedded once. About 16 KB each; 0 disables. +# AUTO_ONTOLOGY_QUERY_EMBED_CACHE_SIZE=1024 # --- Entity extraction tuning --- # Max entities the LLM is asked to extract per question, for SQL generation From 9ea9193cb6af14e7593c2c2c9af70a10c5d88317 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:45:28 -0600 Subject: [PATCH 07/10] Add the SPDX header to the code of conduct The licensing test requires one on every tracked source file, and main has failed it since the file was added. The text is adapted from the Contributor Covenant, so the header names that source and CC-BY-4.0 rather than the NVIDIA Apache-2.0 header the repository's own files use. --- CODE_OF_CONDUCT.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 426bf7b2e..2f21bfabd 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,3 +1,8 @@ + + # Code of Conduct ## Our Pledge From c890d0e790c0b19f1b162df83a6d4fcd58e60b16 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 21:58:53 -0600 Subject: [PATCH 08/10] Address review: bound sqlglot memo growth and the vector store connect The MappingSchema kept per dialect for the life of the catalog map filled sqlglot's per-query memo tables with every table alias any query used: 2,000 parses left 4,000 entries in each. Empty them before each reuse; the normalised catalog mapping, which is what reuse saves, is kept. A test measures every dict on the schema, so a sqlglot release that renames or adds a memo table fails it rather than leaking. PostgresVDB._table_exists connected without a timeout, and the chat worker reaches it while warming, before it reads questions. Bound it at three seconds like the repository's other direct connections. Say plainly that the one-second resend window on a reused LLM session is a heuristic: an endpoint that drops a connection right after reading a request gets it twice. That is kept on purpose, since without it the lost race fails the call instead, and a test now pins the behaviour. Make the unpackable-vector cache test wait until both waiters block on the owner's result before releasing it, so a late thread cannot become a second owner. --- .../catalog/parsers/sqlglot_extractor.py | 21 +++++++++++ .../data_access/tests/test_catalog_schemas.py | 36 ++++++++++++++++++ auto_ontology/utils/llm_invoke.py | 20 ++++++---- auto_ontology/utils/tests/test_llm_invoke.py | 28 ++++++++++++++ .../utils/tests/test_retriever_query_cache.py | 37 +++++++++++++++++-- auto_ontology/vdb/postgres.py | 6 ++- 6 files changed, 137 insertions(+), 11 deletions(-) diff --git a/auto_ontology/catalog/parsers/sqlglot_extractor.py b/auto_ontology/catalog/parsers/sqlglot_extractor.py index ca290b68d..a553c3e73 100644 --- a/auto_ontology/catalog/parsers/sqlglot_extractor.py +++ b/auto_ontology/catalog/parsers/sqlglot_extractor.py @@ -228,6 +228,16 @@ def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]: return schema +# Memo tables sqlglot's MappingSchema fills during a parse, keyed by the query's +# own table expressions (aliases included) and names. They are lookups only: +# clearing one makes the next lookup recompute what it would have returned. +_MAPPING_SCHEMA_QUERY_CACHES = ( + "_normalized_table_cache", + "_find_cache", + "_normalized_name_cache", +) + + def _qualify_schema( all_schemas: dict, schema_dict: dict[str, dict[str, str]], dialect: str ) -> "MappingSchema | dict[str, dict[str, str]]": @@ -238,6 +248,12 @@ def _qualify_schema( untouched. For a :class:`SchemaCatalog` that object is built once per dialect here, by that exact call, and reused; sqlglot only reads from it. Everything else gets the mapping, as before. + + A reused one has its per-query memo tables emptied first. sqlglot keys them + by the parsed query's table expressions, so a schema kept for the life of a + worker would otherwise hold an entry for every alias any query ever used: + 2,000 parses left 4,000 entries in each. What is worth keeping -- the + catalog mapping, normalised once at construction -- is not among them. """ if not isinstance(all_schemas, SchemaCatalog) or not schema_dict: return schema_dict @@ -245,6 +261,11 @@ def _qualify_schema( if cached is None: cached = MappingSchema(schema_dict, dialect=dialect) all_schemas._qualify_schemas[dialect] = cached + else: + for name in _MAPPING_SCHEMA_QUERY_CACHES: + memo = getattr(cached, name, None) + if isinstance(memo, dict): + memo.clear() return cached diff --git a/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py b/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py index fcd07e36e..8e3c3fa2f 100644 --- a/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py +++ b/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py @@ -280,3 +280,39 @@ def test_column_index_follows_a_replaced_frame(catalog: _Catalog) -> None: ].copy() assert schema.column_names_by_table()["customers"] == ("customer_id",) + + +def test_reused_sqlglot_schema_does_not_grow_with_parses(catalog: _Catalog) -> None: + """A worker parses for days; per-query lookups must not accumulate. + + Measured over every dict sqlglot keeps on the schema except the catalog + mapping itself, so a later sqlglot that renames or adds a memo table fails + here instead of leaking quietly. + """ + shared = graph_schemas.get_catalog_schemas() + + def parse(index: int) -> None: + sql = ( + f"SELECT o{index}.total, c{index}.name FROM orders o{index} " + f"JOIN customers c{index} ON o{index}.customer_id = c{index}.customer_id" + ) + with graph_schemas.catalog_schemas_lock: + extract_tables_and_columns( + sql=sql, dialects=["postgres"], all_schemas=shared + ) + + def retained() -> int: + schema = shared._qualify_schemas["postgres"] + return sum( + len(value) + for name, value in vars(schema).items() + if isinstance(value, dict) and name not in ("mapping", "udf_mapping") + ) + + for index in range(5): + parse(index) + after_a_few = retained() + for index in range(5, 300): + parse(index) + + assert retained() == after_a_few diff --git a/auto_ontology/utils/llm_invoke.py b/auto_ontology/utils/llm_invoke.py index ada9ab657..b7f9ad1f3 100644 --- a/auto_ontology/utils/llm_invoke.py +++ b/auto_ontology/utils/llm_invoke.py @@ -213,13 +213,19 @@ class _TimeoutSession(_requests.Session): * Connections idle for longer than ``max_idle`` are closed before the next request, so one the far end has probably dropped is not written to. - * A request on a reused session that fails to connect *immediately* is - sent once more on a new connection. Immediately is the whole condition: - a connection that was already closed fails within a round trip, whereas - a failure seconds in means the endpoint received the request and broke - off while answering -- the failure a fresh session would have had too, - which is left to the caller's retry policy exactly as before. Timeouts - are never resent, whichever exception they arrive as. + * A request on a reused session that fails with a connection error within + a second is sent once more on a new connection. urllib3 already + discards a pooled connection the far end has visibly closed; this covers + the far end closing it in the moment between that check and the write. + The window is a heuristic, not proof the request was never delivered: + an endpoint that reads a request and drops the connection within a + second gets it twice. That is the deliberate side to err on. Without the + resend, the lost race reaches the caller as an error its retry policy + does not retry, and the call falls back to another model or returns + nothing -- a worse answer, where the resend costs one generation. A + failure later than a second is left to the caller exactly as a fresh + session's would be, and timeouts are never resent, whichever exception + they arrive as. * No cookie is ever stored. A one-shot session never sent any, and one set by a load balancer would otherwise pin every later call to one backend. """ diff --git a/auto_ontology/utils/tests/test_llm_invoke.py b/auto_ontology/utils/tests/test_llm_invoke.py index a6c13ff3c..03f6e2801 100644 --- a/auto_ontology/utils/tests/test_llm_invoke.py +++ b/auto_ontology/utils/tests/test_llm_invoke.py @@ -515,6 +515,9 @@ def __init__(self) -> None: endpoint = self self.connections: list[tuple[str, int]] = [] self.cookies: list[str | None] = [] + # When set, the next request is read in full and recorded, and the + # connection is then closed without a response. + self.drop_next = False class Handler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -523,6 +526,10 @@ def do_POST(self) -> None: # noqa: N802 — http.server's name self.rfile.read(int(self.headers.get("Content-Length") or 0)) endpoint.connections.append(self.client_address) endpoint.cookies.append(self.headers.get("Cookie")) + if endpoint.drop_next: + endpoint.drop_next = False + self.close_connection = True + return body = b"{}" self.send_response(200) self.send_header("Content-Type", "application/json") @@ -636,6 +643,27 @@ def dropped_after_thirty_seconds() -> Exception: assert session.sends == 2 +def test_a_request_dropped_right_after_delivery_is_sent_twice( + endpoint: _Endpoint, +) -> None: + """The documented cost of the resend: the window cannot see delivery. + + The endpoint reads the whole request and closes without answering, inside + the window, which is indistinguishable from a connection closed just + before the write. The request is delivered twice and the call succeeds, + where without the resend it would have failed outright. + """ + session = llm_invoke._TimeoutSession(5) + session.post(endpoint.url, json={}) + endpoint.drop_next = True + + response = session.post(endpoint.url, json={"q": 1}) + + assert response.status_code == 200 + assert len(endpoint.connections) == 3 + assert len(set(endpoint.connections)) == 2 + + def test_first_request_connection_failure_is_not_retried(endpoint: _Endpoint) -> None: """Nothing was pooled yet, so the failure is the endpoint's, not staleness.""" session = _FlakySession([requests.exceptions.ConnectionError("refused")]) diff --git a/auto_ontology/utils/tests/test_retriever_query_cache.py b/auto_ontology/utils/tests/test_retriever_query_cache.py index 0deb4af59..e53982e5b 100644 --- a/auto_ontology/utils/tests/test_retriever_query_cache.py +++ b/auto_ontology/utils/tests/test_retriever_query_cache.py @@ -13,6 +13,7 @@ from __future__ import annotations import threading +from concurrent.futures import Future from typing import Any import pytest @@ -171,8 +172,31 @@ def search() -> None: assert all("503" in str(error) for error in errors) -def test_a_vector_that_cannot_be_packed_is_passed_on_and_strands_nobody() -> None: +class _CountingFuture(Future): + """A Future that counts the threads blocked on its result.""" + + def __init__(self) -> None: + super().__init__() + self.waiting = 0 + self._count_lock = threading.Lock() + + def result(self, timeout: float | None = None) -> Any: + with self._count_lock: + self.waiting += 1 + return super().result(timeout) + + +def test_a_vector_that_cannot_be_packed_is_passed_on_and_strands_nobody( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Whatever the endpoint returned still reaches the store, and every waiter.""" + futures: list[_CountingFuture] = [] + + def make_future() -> _CountingFuture: + futures.append(_CountingFuture()) + return futures[-1] + + monkeypatch.setattr(retriever_mod, "Future", make_future) cache = _QueryEmbeddingCache(8) gate = threading.Event() odd = [[0.1, 0.2]] # nested: not a flat list of numbers @@ -190,14 +214,21 @@ def compute() -> list: ] for thread in threads: thread.start() - while not calls: - pass + # Release the owner only once the other two are blocked on its result, so + # none of them can arrive late and become an owner itself. + deadline = threading.Event() + for _ in range(500): + if futures and futures[0].waiting == 2: + break + deadline.wait(0.01) + assert futures and futures[0].waiting == 2 gate.set() for thread in threads: thread.join(5) assert not any(thread.is_alive() for thread in threads) assert results == [odd, odd, odd] + assert len(calls) == 1 # Not kept, so the next request asks again instead of reusing it. assert cache.get(("k",), compute) == odd assert len(calls) == 2 diff --git a/auto_ontology/vdb/postgres.py b/auto_ontology/vdb/postgres.py index 2733a7dce..dccab5606 100644 --- a/auto_ontology/vdb/postgres.py +++ b/auto_ontology/vdb/postgres.py @@ -215,7 +215,11 @@ def _get_engine(self) -> PGEngine: return self._engine def _table_exists(self) -> bool: - with psycopg.connect(self.connection_string) as conn: + # Bounded like every other direct Postgres connection in the repo: a + # blackholed database drops packets rather than refusing them, and an + # unbounded connect would hang the caller instead of failing. The + # chat worker reaches this from its warm-up, before reading questions. + with psycopg.connect(self.connection_string, connect_timeout=3) as conn: with conn.cursor() as cur: cur.execute( """ From b7666f530296804d889c46e37cfeae643c22bc43 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Thu, 12 Nov 2026 22:12:35 -0600 Subject: [PATCH 09/10] Address review: keep vector store warming read-only under a race warm() checked that the collection existed and then called _get_store(), which checked again and created the collection if it had vanished in between. Bind with the create path switched off, so existence is checked once and a dropped collection makes the bind fail rather than come back. Bound the existence query as well as its connect: a statement timeout for a server that stops answering, and TCP keepalives for a peer that disappears, so warming cannot hold the worker indefinitely. --- auto_ontology/vdb/postgres.py | 41 ++++++++++++++++--------- auto_ontology/vdb/tests/test_records.py | 13 ++++++++ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/auto_ontology/vdb/postgres.py b/auto_ontology/vdb/postgres.py index dccab5606..7b588b773 100644 --- a/auto_ontology/vdb/postgres.py +++ b/auto_ontology/vdb/postgres.py @@ -215,11 +215,21 @@ def _get_engine(self) -> PGEngine: return self._engine def _table_exists(self) -> bool: - # Bounded like every other direct Postgres connection in the repo: a - # blackholed database drops packets rather than refusing them, and an - # unbounded connect would hang the caller instead of failing. The - # chat worker reaches this from its warm-up, before reading questions. - with psycopg.connect(self.connection_string, connect_timeout=3) as conn: + # Bounded at every stage, since the chat worker reaches this from its + # warm-up before it reads questions: the connect (a blackholed database + # drops packets rather than refusing them), the query (a server that + # stops answering after connect), and a peer that silently disappears + # (TCP keepalives notice in about five seconds). A one-row catalog + # lookup has no business taking longer than three seconds anywhere. + with psycopg.connect( + self.connection_string, + connect_timeout=3, + keepalives=1, + keepalives_idle=3, + keepalives_interval=1, + keepalives_count=2, + options="-c statement_timeout=3000", + ) as conn: with conn.cursor() as cur: cur.execute( """ @@ -249,17 +259,20 @@ def _ensure_schema(self) -> None: ) conn.commit() - def _get_store(self) -> Optional[PGVectorStore]: + def _get_store(self, *, create_missing: bool = True) -> Optional[PGVectorStore]: """Return the vector store, creating the table on first write. - If the table doesn't exist and ``vector_size`` is not provided (e.g. - on read paths), returns ``None`` so callers can short-circuit. + With ``create_missing=False`` a collection that does not exist is left + alone and ``None`` is returned, for callers that must never write. """ if self._store is not None: return self._store + exists = self._table_exists() + if not exists and not create_missing: + return None engine = self._get_engine() - if not self._table_exists(): + if not exists: self._ensure_schema() engine.init_vectorstore_table( table_name=self.collection_name, @@ -351,13 +364,11 @@ def warm(self) -> bool: Read-only, unlike :meth:`create_index`: a collection that does not exist yet is left that way, and the first write creates it as before. - Returns whether the store is now ready. + Existence is checked once, by the bind itself, so a collection dropped + a moment earlier cannot slip past a separate check and be recreated; + at worst binding fails and raises. Returns whether the store is ready. """ - if self._store is not None: - return True - if not self._table_exists(): - return False - return self._get_store() is not None + return self._get_store(create_missing=False) is not None # ------------------------------------------------------------------ # Read / write diff --git a/auto_ontology/vdb/tests/test_records.py b/auto_ontology/vdb/tests/test_records.py index 4db83b5db..2338176b0 100644 --- a/auto_ontology/vdb/tests/test_records.py +++ b/auto_ontology/vdb/tests/test_records.py @@ -185,3 +185,16 @@ def test_warming_never_creates_a_collection(vdb: Any) -> None: assert vdb.warm() is True [hits] = vdb.retrieval([_unit(0)], top_k=1) assert hits[0]["metadata"]["id"] == "orders" + + +def test_warming_a_collection_dropped_after_its_check_does_not_recreate_it( + vdb: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """The existence check says yes, then the table is gone by the time of binding.""" + monkeypatch.setattr(vdb, "_table_exists", lambda: True) + + with pytest.raises(Exception): + vdb.warm() + + monkeypatch.undo() + assert vdb._table_exists() is False From 39e344625d93af3169d7b9a35b4cb078abd43a1d Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Sat, 14 Nov 2026 17:34:06 -0600 Subject: [PATCH 10/10] Read a mocked schema's columns from its frame in SQL extraction A MagicMock schema answers to column_names_by_table with another mock, so the extractor read it as a schema with no columns and ignored its columns_df. Main's new extractor tests mock schemas this way. Only a real mapping is taken from the accessor now. --- .../catalog/parsers/sqlglot_extractor.py | 9 +++++- .../catalog/tests/test_sqlglot_extractor.py | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/auto_ontology/catalog/parsers/sqlglot_extractor.py b/auto_ontology/catalog/parsers/sqlglot_extractor.py index 4deb6ec75..8d88df202 100644 --- a/auto_ontology/catalog/parsers/sqlglot_extractor.py +++ b/auto_ontology/catalog/parsers/sqlglot_extractor.py @@ -188,10 +188,17 @@ def _schema_column_index(schema: Any) -> dict[str, tuple[str, ...]]: A :class:`~auto_ontology.catalog.model.Schema` keeps this itself; anything else that merely looks like one (a ``columns_df`` and nothing more) has it derived here, the same way. + + Only a real mapping is taken from the accessor. A ``MagicMock`` standing in + for a schema answers to any attribute name, so it has a callable + ``column_names_by_table`` that returns another mock; trusting that would + read the mock as a schema with no columns and ignore its ``columns_df``. """ accessor = getattr(schema, "column_names_by_table", None) if callable(accessor): - return accessor() + index = accessor() + if isinstance(index, dict): + return index df = getattr(schema, "columns_df", None) if df is None or df.empty: return {} diff --git a/auto_ontology/catalog/tests/test_sqlglot_extractor.py b/auto_ontology/catalog/tests/test_sqlglot_extractor.py index f2e2e4727..7fc8d5b4d 100644 --- a/auto_ontology/catalog/tests/test_sqlglot_extractor.py +++ b/auto_ontology/catalog/tests/test_sqlglot_extractor.py @@ -74,3 +74,32 @@ def test_parse_query_single_succeeds_for_count_star() -> None: assert query_obj is not None assert query_obj.get_tables_ids() == ["node-customers"] assert query_obj.get_column_ids() == [] + + +def test_a_mocked_schema_is_read_from_its_columns_frame() -> None: + """A ``MagicMock`` schema's ``columns_df`` is what attributes bare columns. + + A mock answers to any attribute name, so it appears to have the + ``column_names_by_table`` accessor a real ``Schema`` has, and calling it + returns another mock. Taking that for the column index would parse every + mocked schema as an empty catalog: ``name`` and ``total`` below would be + attributed to no table at all. + """ + schema = MagicMock() + schema.columns_df = pd.DataFrame( + [ + {"table_name": "customers", "column_name": "id"}, + {"table_name": "customers", "column_name": "name"}, + {"table_name": "orders", "column_name": "customer_id"}, + {"table_name": "orders", "column_name": "total"}, + ] + ) + + result = extract_tables_and_columns( + "SELECT name, total FROM customers c JOIN orders o ON c.id = o.customer_id", + ["postgres"], + {"shop": schema}, + ) + + assert result.tables["customers"].columns == {"id", "name"} + assert result.tables["orders"].columns == {"customer_id", "total"}