diff --git a/.env.example b/.env.example index 6aeaa5a7b..aa1770d23 100644 --- a/.env.example +++ b/.env.example @@ -180,6 +180,15 @@ AUTH_SECRET=myverylongsecret123 # Max concurrent LLM requests across all worker threads, to stay under the # hosted endpoint's per-worker request cap (exceeding it surfaces as HTTP 503). # LLM_MAX_INFLIGHT=6 +# How long, in seconds, a connection to the LLM endpoint may sit unused before +# it is replaced rather than reused for the next call. Applies to models served +# through ChatNVIDIA. Lower it if the endpoint closes idle connections sooner. +# LLM_KEEPALIVE_IDLE_S=5 + +# --- Retrieval tuning (auto_ontology/utils/retriever.py) --- +# How many distinct query strings keep their embedding in memory, so a string +# searched for more than once is embedded once. About 16 KB each; 0 disables. +# AUTO_ONTOLOGY_QUERY_EMBED_CACHE_SIZE=1024 # --- Entity extraction tuning --- # Max entities the LLM is asked to extract per question, for SQL generation diff --git a/auto_ontology/catalog/model/schema.py b/auto_ontology/catalog/model/schema.py index 73f82a280..349ca940f 100644 --- a/auto_ontology/catalog/model/schema.py +++ b/auto_ontology/catalog/model/schema.py @@ -92,12 +92,15 @@ def __init__( self.columns_df.set_index("id")["full_name"].to_dict() ) - # This is for improving is_column_in_table + # This is for improving is_column_in_table. Read column-wise: one + # `iterrows()` pass builds a Series per row, which at catalog + # scale was half the cost of constructing a Schema. self.columns_map = {} - for _, row in self.columns_df.iterrows(): - if row["table_name_lower"] not in self.columns_map: - self.columns_map[row["table_name_lower"]] = {} - self.columns_map[row["table_name_lower"]][row["column_name_lower"]] = ( + for table_name_lower, column_name_lower in zip( + self.columns_df["table_name_lower"], + self.columns_df["column_name_lower"], + ): + self.columns_map.setdefault(table_name_lower, {})[column_name_lower] = ( True ) @@ -151,6 +154,52 @@ def reset_columns_props(self): axis=1, ) + def column_names_by_table(self) -> dict[str, tuple[str, ...]]: + """Lower-cased column names per lower-cased table name, in row order. + + This is what SQL extraction needs from the catalog on every parse: the + column set of each table, to hand to sqlglot's ``qualify`` and to + attribute bare column references. Reading it off ``columns_df`` row by + row costs a pandas Series per column of the *whole* schema, per parse, + which made a parse scale with the catalog instead of with the query. + It is derived once and kept for as long as ``columns_df`` is the same + frame with the same number of rows. + + Names are lower-cased with ``str(...).lower()`` and nothing else, on + purpose: unlike ``columns_map`` they keep any surrounding quotes, which + is the spelling the extractor has always matched against. + """ + df = self.columns_df + if df is None or df.empty: + return {} + # The frame itself is held, not its id(): an id can be handed to a new + # object once the old one is collected, and would then vouch for an + # index read from a frame that no longer exists. + cached = getattr(self, "_column_names_by_table", None) + if cached is not None and cached[0] is df and cached[1] == len(df): + return cached[2] + grouped: dict[str, list[str]] = {} + for table_name, column_name in zip(df["table_name"], df["column_name"]): + grouped.setdefault(str(table_name).lower(), []).append( + str(column_name).lower() + ) + index = {table: tuple(columns) for table, columns in grouped.items()} + self._column_names_by_table = (df, len(df), index) + return index + + def forget_nodes(self) -> None: + """Drop the table and column nodes built by earlier lookups. + + ``get_table_node`` and ``get_column_node`` build a node on first use and + keep it. A schema that is parsed against repeatedly calls this between + parses so that each one resolves names as the first did; the frames, + ``columns_map`` and ``id_to_node`` it resolves them from are untouched. + """ + self.table_nodes = {} + self.column_nodes = {} + self.tables_to_columns = {} + self.tables_columns_pos = {} + def get_schema_name(self): return self.schema_name diff --git a/auto_ontology/catalog/parsers/sqlglot_extractor.py b/auto_ontology/catalog/parsers/sqlglot_extractor.py index 6b7a021fb..8d88df202 100644 --- a/auto_ontology/catalog/parsers/sqlglot_extractor.py +++ b/auto_ontology/catalog/parsers/sqlglot_extractor.py @@ -27,11 +27,13 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import Any import sqlglot from sqlglot import exp from sqlglot.errors import ParseError, TokenError from sqlglot.optimizer.qualify import qualify +from sqlglot.schema import MappingSchema class SQLSyntaxError(ValueError): @@ -151,6 +153,61 @@ def _alias_to_table_map( return mapping +class SchemaCatalog(dict): + """A ``{schema_name: Schema}`` map that is parsed against more than once. + + Extraction derives two things from the catalog that depend on the catalog + alone, not on the SQL: the flat ``{table: {column: type}}`` mapping + ``qualify`` reads, and the ``MappingSchema`` sqlglot normalises that + mapping into for one dialect. Both are linear in the number of columns in + the *whole* catalog, so rebuilding them per statement makes every parse + pay for every table nobody asked about. + + A plain ``dict`` has nowhere to keep them. Wrapping the map in this class + is how a caller says the same catalog will be reused, and that it will not + add, drop or swap a schema in place afterwards -- build a new + ``SchemaCatalog`` for a changed catalog instead. A plain ``dict`` still + works everywhere and is simply derived from each time. + + Not safe for concurrent parses on its own: these caches, like the lazily + built nodes on each ``Schema``, are filled on first use. Callers sharing + one across threads serialise the parse. + """ + + __slots__ = ("_qualify_columns", "_qualify_schemas") + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._qualify_columns: dict[str, dict[str, str]] | None = None + self._qualify_schemas: dict[str, MappingSchema] = {} + + +def _schema_column_index(schema: Any) -> dict[str, tuple[str, ...]]: + """``{table: (column, ...)}`` for one catalog schema, names lower-cased. + + A :class:`~auto_ontology.catalog.model.Schema` keeps this itself; anything + else that merely looks like one (a ``columns_df`` and nothing more) has it + derived here, the same way. + + Only a real mapping is taken from the accessor. A ``MagicMock`` standing in + for a schema answers to any attribute name, so it has a callable + ``column_names_by_table`` that returns another mock; trusting that would + read the mock as a schema with no columns and ignore its ``columns_df``. + """ + accessor = getattr(schema, "column_names_by_table", None) + if callable(accessor): + index = accessor() + if isinstance(index, dict): + return index + df = getattr(schema, "columns_df", None) + if df is None or df.empty: + return {} + grouped: dict[str, list[str]] = {} + for table_name, column_name in zip(df["table_name"], df["column_name"]): + grouped.setdefault(str(table_name).lower(), []).append(str(column_name).lower()) + return {table: tuple(columns) for table, columns in grouped.items()} + + def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]: """Build a flat ``{table: {col: "TEXT"}}`` dict for sqlglot's qualify pass. @@ -161,18 +218,64 @@ def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]: Multi-schema disambiguation is handled separately via ``alias_map`` and ``source_table_names`` (which use fully-qualified keys). """ + if isinstance(all_schemas, SchemaCatalog) and ( + all_schemas._qualify_columns is not None + ): + return all_schemas._qualify_columns + schema: dict[str, dict[str, str]] = {} for s in all_schemas.values(): - df = s.columns_df - if df is None or df.empty: - continue - for _, row in df.iterrows(): - tbl = str(row["table_name"]).lower() - col = str(row["column_name"]).lower() - schema.setdefault(tbl, {})[col] = "TEXT" + for tbl, columns in _schema_column_index(s).items(): + table_columns = schema.setdefault(tbl, {}) + for col in columns: + table_columns[col] = "TEXT" + + if isinstance(all_schemas, SchemaCatalog): + all_schemas._qualify_columns = schema return schema +# Memo tables sqlglot's MappingSchema fills during a parse, keyed by the query's +# own table expressions (aliases included) and names. They are lookups only: +# clearing one makes the next lookup recompute what it would have returned. +_MAPPING_SCHEMA_QUERY_CACHES = ( + "_normalized_table_cache", + "_find_cache", + "_normalized_name_cache", +) + + +def _qualify_schema( + all_schemas: dict, schema_dict: dict[str, dict[str, str]], dialect: str +) -> "MappingSchema | dict[str, dict[str, str]]": + """What to hand ``qualify`` as its schema for *dialect*. + + ``qualify`` turns a mapping into ``MappingSchema(mapping, dialect=dialect)`` + before it does anything else, and passes an existing schema object through + untouched. For a :class:`SchemaCatalog` that object is built once per + dialect here, by that exact call, and reused; sqlglot only reads from it. + Everything else gets the mapping, as before. + + A reused one has its per-query memo tables emptied first. sqlglot keys them + by the parsed query's table expressions, so a schema kept for the life of a + worker would otherwise hold an entry for every alias any query ever used: + 2,000 parses left 4,000 entries in each. What is worth keeping -- the + catalog mapping, normalised once at construction -- is not among them. + """ + if not isinstance(all_schemas, SchemaCatalog) or not schema_dict: + return schema_dict + cached = all_schemas._qualify_schemas.get(dialect) + if cached is None: + cached = MappingSchema(schema_dict, dialect=dialect) + all_schemas._qualify_schemas[dialect] = cached + else: + for name in _MAPPING_SCHEMA_QUERY_CACHES: + memo = getattr(cached, name, None) + if isinstance(memo, dict): + memo.clear() + return cached + + _BINARY_CMP = (exp.EQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.NEQ) _OP_SYMBOL: dict[type, str] = { @@ -692,7 +795,7 @@ def extract_tables_and_columns( qualified = qualify( statement, dialect=dialect, - schema=schema_dict, + schema=_qualify_schema(all_schemas, schema_dict, dialect), qualify_columns=True, validate_qualify_columns=False, expand_stars=bool(schema_dict), @@ -718,13 +821,11 @@ def extract_tables_and_columns( if unresolved and all_schemas: col_to_tables: dict[str, list[str]] = {} for schema_name, s in all_schemas.items(): - df = s.columns_df - if df is None or df.empty: - continue skey = schema_name.lower() - for _, row in df.iterrows(): - col_n = str(row["column_name"]).lower() - tbl_n = str(row["table_name"]).lower() + # Per table, not per column: which source table a catalog table + # stands for is decided by its name alone, so only the columns of + # tables this query actually reads are ever walked. + for tbl_n, columns in _schema_column_index(s).items(): # Try the qualified name first (schema.table); fall back to bare # table name for SQL that doesn't prefix tables with a schema. qualified_name = f"{skey}.{tbl_n}" @@ -733,8 +834,11 @@ def extract_tables_and_columns( if qualified_name in source_table_names else (tbl_n if tbl_n in source_table_names else None) ) - if matched and matched not in col_to_tables.get(col_n, []): - col_to_tables.setdefault(col_n, []).append(matched) + if not matched: + continue + for col_n in columns: + if matched not in col_to_tables.get(col_n, []): + col_to_tables.setdefault(col_n, []).append(matched) for col_name in unresolved: candidates = col_to_tables.get(col_name, []) diff --git a/auto_ontology/catalog/tests/test_sqlglot_extractor.py b/auto_ontology/catalog/tests/test_sqlglot_extractor.py index f2e2e4727..7fc8d5b4d 100644 --- a/auto_ontology/catalog/tests/test_sqlglot_extractor.py +++ b/auto_ontology/catalog/tests/test_sqlglot_extractor.py @@ -74,3 +74,32 @@ def test_parse_query_single_succeeds_for_count_star() -> None: assert query_obj is not None assert query_obj.get_tables_ids() == ["node-customers"] assert query_obj.get_column_ids() == [] + + +def test_a_mocked_schema_is_read_from_its_columns_frame() -> None: + """A ``MagicMock`` schema's ``columns_df`` is what attributes bare columns. + + A mock answers to any attribute name, so it appears to have the + ``column_names_by_table`` accessor a real ``Schema`` has, and calling it + returns another mock. Taking that for the column index would parse every + mocked schema as an empty catalog: ``name`` and ``total`` below would be + attributed to no table at all. + """ + schema = MagicMock() + schema.columns_df = pd.DataFrame( + [ + {"table_name": "customers", "column_name": "id"}, + {"table_name": "customers", "column_name": "name"}, + {"table_name": "orders", "column_name": "customer_id"}, + {"table_name": "orders", "column_name": "total"}, + ] + ) + + result = extract_tables_and_columns( + "SELECT name, total FROM customers c JOIN orders o ON c.id = o.customer_id", + ["postgres"], + {"shop": schema}, + ) + + assert result.tables["customers"].columns == {"id", "name"} + assert result.tables["orders"].columns == {"customer_id", "total"} diff --git a/auto_ontology/dal/datasources.py b/auto_ontology/dal/datasources.py index 369f2d72c..21111937a 100644 --- a/auto_ontology/dal/datasources.py +++ b/auto_ontology/dal/datasources.py @@ -24,9 +24,11 @@ import pandas as pd from sqlalchemy import ( ColumnElement, + Numeric, Text, and_, case, + cast, column, distinct, func, @@ -252,6 +254,93 @@ def fetch_schemas_by_ids( return [dict(r) for r in store().query_read(statement)] +def _schema_row_hash() -> ColumnElement: + """One 64-bit hash per row of the ``fetch_schemas_by_ids`` result. + + Over exactly the seven fields that query selects, and nothing else: a + description edit or a new sample value leaves it alone, while a renamed + column, a retyped one, or a table added, dropped or moved does not. + + ``data_type`` goes last because it is the only nullable field and + ``concat_ws`` skips nulls: at the end, a missing type shortens the string + rather than shifting another field into its place. + """ + return func.hashtextextended( + func.concat_ws( + # The unit separator, which no identifier contains, so two rows + # cannot hash alike by splitting the same text at different points. + literal("\x1f"), + s.catalog_column.c.id, + s.catalog_column.c.name, + s.catalog_table.c.id, + s.catalog_table.c.name, + s.catalog_schema.c.name, + s.catalog_database.c.name, + s.catalog_column.c.data_type, + ), + 0, + ) + + +def _schema_fingerprint(row_count: int, digest: Any) -> str: + """How a row count and a sum of row hashes are written down. + + One place, because the two functions below must agree to the character: + one gets the sum from Postgres as a ``Decimal``, the other adds the hashes + up itself. + """ + return f"{int(row_count)}:{int(digest)}" + + +def fetch_catalog_schema_fingerprint() -> str: + """A digest that changes whenever ``fetch_schemas_by_ids()`` would. + + Lets a caller keep what it built from the whole-catalog rows and learn, + for the price of one aggregate, whether it is still current. The rows are + hashed where they live, so nothing but the answer crosses the wire. + + The sum makes it independent of row order, and the count keeps an added + row from being cancelled by a removed one whose hashes happen to offset. + Compare it only with the fingerprint + :func:`fetch_catalog_schema_snapshot` returned alongside the rows. + """ + statement = select( + func.count().label("row_count"), + # Numeric: summing bigint hashes overflows bigint long before the + # catalog is large. + func.coalesce(func.sum(cast(_schema_row_hash(), Numeric)), 0).label("digest"), + ).select_from(_catalog_join()) + + row = store().query_read(statement)[0] + return _schema_fingerprint(row["row_count"], row["digest"]) + + +def fetch_catalog_schema_snapshot() -> tuple[str, list[dict[str, str]]]: + """Every row ``fetch_schemas_by_ids()`` returns, with their fingerprint. + + The fingerprint is computed from the rows returned, which one statement + read from one snapshot of the database. Reading the fingerprint and the + rows separately cannot promise that: a commit landing between the two + pairs a fingerprint with rows it does not describe, and because the + fingerprint is a function of content, a catalog that later returns to the + earlier state would match it again and vouch for the wrong rows. + """ + statement = select( + s.catalog_column.c.name.label("column_name"), + s.catalog_column.c.id.label("column_id"), + s.catalog_table.c.name.label("table_name"), + s.catalog_table.c.id.label("table_id"), + s.catalog_database.c.name.label("database_name"), + s.catalog_schema.c.name.label("table_schema"), + s.catalog_column.c.data_type, + _schema_row_hash().label("row_hash"), + ).select_from(_catalog_join()) + + rows = [dict(r) for r in store().query_read(statement)] + digest = sum(row.pop("row_hash") for row in rows) + return _schema_fingerprint(len(rows), digest), rows + + # --------------------------------------------------------------------------- # Table # --------------------------------------------------------------------------- diff --git a/auto_ontology/dal/tests/dal_surface.json b/auto_ontology/dal/tests/dal_surface.json index 8f4fd2a54..3373ebbad 100644 --- a/auto_ontology/dal/tests/dal_surface.json +++ b/auto_ontology/dal/tests/dal_surface.json @@ -55,6 +55,8 @@ "fetch_all_schema_ids": "() -> 'list[str]'", "fetch_all_tables_without_term": "(database_name: 'str | None' = None) -> 'list[dict[str, Any]]'", "fetch_bridge_table_candidates": "(database_name: 'str') -> 'list[dict[str, Any]]'", + "fetch_catalog_schema_fingerprint": "() -> 'str'", + "fetch_catalog_schema_snapshot": "() -> 'tuple[str, list[dict[str, str]]]'", "fetch_col_table_contexts": "(col_ids: 'list[str]') -> 'dict[str, dict[str, str]]'", "fetch_columns_for_table": "(table_id: 'str', *, skip: 'int' = 0, limit: 'int | None' = None) -> 'dict[str, Any] | None'", "fetch_databases": "(zone_ids: 'list[str] | None' = None) -> 'list[dict[str, Any]]'", diff --git a/auto_ontology/dal/tests/test_datasources_semantic.py b/auto_ontology/dal/tests/test_datasources_semantic.py index 8e2cfc38f..58b2f79dd 100644 --- a/auto_ontology/dal/tests/test_datasources_semantic.py +++ b/auto_ontology/dal/tests/test_datasources_semantic.py @@ -820,3 +820,81 @@ def test_an_fk_pointing_outside_the_catalog_disqualifies(world) -> None: assert "tag_link" not in [ r["table_name"] for r in d.fetch_bridge_table_candidates(world.prefix) ] + + +# --------------------------------------------------------------------------- +# fetch_catalog_schema_fingerprint +# --------------------------------------------------------------------------- + + +def _row_set(rows: list[dict]) -> set[tuple]: + return {tuple(sorted(row.items(), key=lambda item: item[0])) for row in rows} + + +def test_fingerprint_tracks_exactly_what_the_schema_rows_contain(world: World) -> None: + """It must move with every field ``fetch_schemas_by_ids`` returns, and no other. + + Whatever is built from those rows is kept for as long as the fingerprint + holds, so a change it misses is a stale catalog served to SQL validation. + """ + seen = [d.fetch_catalog_schema_fingerprint()] + + def changed() -> bool: + seen.append(d.fetch_catalog_schema_fingerprint()) + # The fingerprint that comes back with the rows is the one a cached + # map is filed under, so it has to be the same string, every time. + snapshot_fingerprint, rows = d.fetch_catalog_schema_snapshot() + assert snapshot_fingerprint == seen[-1] + assert _row_set(rows) == _row_set(d.fetch_schemas_by_ids()) + return seen[-1] != seen[-2] + + assert not changed(), "two reads of an untouched catalog must agree" + + world.table("refunds", table_type="base table") + assert not changed(), "a table with no columns contributes no rows" + + refund_id = world.column("refunds", "refund_id", 1) + assert changed(), "an added column" + + store().query_write( + s.catalog_column.update() + .where(s.catalog_column.c.id == refund_id) + .values(description="Refund identifier", sample_values='["r1"]') + ) + assert not changed(), "a description or sample values are not in the rows" + + store().query_write( + s.catalog_column.update() + .where(s.catalog_column.c.id == refund_id) + .values(data_type="bigint") + ) + assert changed(), "a type going from NULL to a value" + + store().query_write( + s.catalog_column.update() + .where(s.catalog_column.c.id == refund_id) + .values(name="refund_key") + ) + assert changed(), "a renamed column" + + store().query_write( + s.catalog_table.update() + .where(s.catalog_table.c.id == world.tables["refunds"]) + .values(name="returns") + ) + assert changed(), "a renamed table" + + store().query_write( + s.catalog_schema.update() + .where(s.catalog_schema.c.id == world.schema) + .values(name="store") + ) + assert changed(), "a renamed schema" + + store().query_write( + s.catalog_column.delete().where(s.catalog_column.c.id == refund_id) + ) + assert changed(), "a dropped column" + assert len(set(seen)) == len(seen) - 3, ( + "each change gave a fingerprint not seen before" + ) diff --git a/auto_ontology/retrieval/data_access/graph_schemas.py b/auto_ontology/retrieval/data_access/graph_schemas.py index bbfa96552..6bcd04583 100644 --- a/auto_ontology/retrieval/data_access/graph_schemas.py +++ b/auto_ontology/retrieval/data_access/graph_schemas.py @@ -15,6 +15,7 @@ from __future__ import annotations import logging +import threading import time import pandas as pd @@ -22,9 +23,12 @@ from auto_ontology.catalog.model import CatalogNode from auto_ontology.catalog.constants import Labels from auto_ontology.catalog.model import Schema +from auto_ontology.catalog.parsers.sqlglot_extractor import SchemaCatalog from auto_ontology.dal.datasources import ( fetch_all_schema_ids, + fetch_catalog_schema_fingerprint, + fetch_catalog_schema_snapshot, fetch_schemas_by_ids, fetch_item_by_id, ) @@ -32,11 +36,117 @@ logger = logging.getLogger(__name__) __all__ = [ + "catalog_schemas_lock", "fetch_all_schema_ids", "fetch_item_by_id", + "forget_parsed_nodes", + "get_catalog_schemas", "get_schemas_by_ids", + "reset_catalog_schemas", ] +#: Held while parsing against what :func:`get_catalog_schemas` returns. +#: +#: That map is shared by every caller in the process, and both it and the +#: ``Schema`` objects in it fill caches on first use (table and column nodes, +#: the mapping ``qualify`` reads). Two threads filling one at once would each +#: build their own copy of a node and disagree about which is *the* node for a +#: table. A parse against the shared map takes milliseconds, so one lock around +#: it costs nothing next to the model call that follows. Reentrant, so a caller +#: already holding it can still ask for the map. +catalog_schemas_lock = threading.RLock() + +# The whole-catalog map and the fingerprint of the rows it was built from. +_catalog_snapshot: tuple[str, SchemaCatalog] | None = None + + +def reset_catalog_schemas() -> None: + """Forget the cached catalog map, so the next caller builds a fresh one.""" + global _catalog_snapshot + with catalog_schemas_lock: + _catalog_snapshot = None + + +def forget_parsed_nodes(schemas: dict) -> None: + """Start a parse against the shared map with no nodes left from the last one. + + A ``Schema`` builds a table or column node the first time it is asked for + one and answers from memory after that. Kept across parses, that memory + makes a lookup depend on what was parsed before it: a column whose stored + name carries surrounding quotes fails its first lookup and succeeds on + every later one, so the columns validation reports for the same SQL would + differ between the first question and the second. + + Every validation used to get schemas nobody had parsed against. Dropping + the nodes gives each parse the same starting point and keeps what is + expensive: the frames, the column index and the id map never change after + the build. Only the map from :func:`get_catalog_schemas` is touched; a + map built for one call has no earlier parse to forget. + """ + if not isinstance(schemas, SchemaCatalog): + return + for schema in schemas.values(): + schema.forget_nodes() + + +def get_catalog_schemas() -> dict: + """The schema map for the whole catalog, rebuilt only when the catalog changes. + + Equivalent to ``get_schemas_by_ids(fetch_all_schema_ids())``, which is what + SQL validation used to run before every parse. That reads every column of + every database and turns the rows into ``Schema`` objects; it is linear in + the size of the catalog, and a question is validated once per SQL attempt, + up to eight times. Measured without a database in the way: 30 ms for a + 440-column catalog, 1.5 s at 50,000 columns, 4.6 s at 150,000 -- per + attempt, on the interpreter lock, so concurrent questions queued behind it. + + Here the rows are read only when + :func:`~auto_ontology.dal.datasources.fetch_catalog_schema_fingerprint` + differs from the fingerprint of the rows the cached map was built from. + The check is a single aggregate, and it is made on every call: a table + ingested a moment ago is visible to the very next validation, exactly as + it was before. + + The map is stored under the fingerprint that came back *with its rows*, + from the same statement, never under the one read beforehand. A commit + between two separate reads would otherwise file new rows under an old + fingerprint, and a catalog that later returned to that old state would be + served the wrong map for as long as it stayed there. + + When the fingerprint cannot be read at all, the map is built as it always + was and not kept. Anything raised while building propagates and nothing is + cached, so an empty or unreachable catalog fails the same way it did. + + Hold :data:`catalog_schemas_lock` while parsing against the result, and + call :func:`forget_parsed_nodes` first. + """ + global _catalog_snapshot + + try: + fingerprint = fetch_catalog_schema_fingerprint() + except Exception: + logger.warning( + "Catalog fingerprint unavailable; building the schema map uncached", + exc_info=True, + ) + return get_schemas_by_ids(fetch_all_schema_ids()) + + with catalog_schemas_lock: + snapshot = _catalog_snapshot + if snapshot is not None and snapshot[0] == fingerprint: + return snapshot[1] + started = time.perf_counter() + rows_fingerprint, rows = fetch_catalog_schema_snapshot() + schemas = SchemaCatalog(_schemas_from_rows(rows)) + _catalog_snapshot = (rows_fingerprint, schemas) + logger.info( + "Catalog schema map %s in %.2fs (%d schema(s))", + "built" if snapshot is None else "rebuilt after a catalog change", + time.perf_counter() - started, + len(schemas), + ) + return schemas + def get_schemas_by_ids(relevant_schemas_ids: list | None = None) -> dict: """Assemble :class:`Schema` objects from catalog data. @@ -47,6 +157,12 @@ def get_schemas_by_ids(relevant_schemas_ids: list | None = None) -> dict: before_get_all = time.time() data_array = fetch_schemas_by_ids(relevant_schemas_ids) logger.info(f"time took to get all data from graph: {time.time() - before_get_all}") + return _schemas_from_rows(data_array) + + +def _schemas_from_rows(data_array: list[dict[str, str]]) -> dict: + """Turn ``fetch_schemas_by_ids``-shaped rows into the ``Schema`` map.""" + before_get_all = time.time() data_df = pd.DataFrame(data_array) dbs = list(data_df["database_name"].unique()) @@ -101,5 +217,7 @@ def get_schemas_by_ids(relevant_schemas_ids: list | None = None) -> dict: logger.info( f"total time it took to create all schemas nodes: {time.time() - before_modify_all}" ) - logger.info(f"total time for get_schemas_by_ids(): {time.time() - before_get_all}") + logger.info( + f"total time to build schemas from rows: {time.time() - before_get_all}" + ) return all_schemas diff --git a/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py b/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py new file mode 100644 index 000000000..8e3c3fa2f --- /dev/null +++ b/auto_ontology/retrieval/data_access/tests/test_catalog_schemas.py @@ -0,0 +1,318 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The whole-catalog schema map is built once per catalog state, never stale. + +SQL validation parses every attempt against it. Rebuilding it each time made a +validation as slow as the catalog is large; keeping it is only safe if a +changed catalog is always noticed, and if what a parse returns from the kept +map is what it returned from a fresh one. +""" + +from __future__ import annotations + +import uuid +from typing import Any + +import pytest + +from auto_ontology.catalog.parsers.sqlglot_extractor import ( + SchemaCatalog, + extract_tables_and_columns, +) +from auto_ontology.catalog.sql_parse import parse_query_single +from auto_ontology.retrieval.data_access import graph_schemas + + +def _rows(tables: dict[str, list[str]], schema: str = "shop") -> list[dict[str, str]]: + """Catalog rows in the shape ``fetch_schemas_by_ids`` returns.""" + rows = [] + for table_name, columns in tables.items(): + table_id = str(uuid.uuid4()) + for column_name in columns: + rows.append( + { + "column_name": column_name, + "column_id": str(uuid.uuid4()), + "table_name": table_name, + "table_id": table_id, + "database_name": "db", + "table_schema": schema, + "data_type": "text", + } + ) + return rows + + +class _Catalog: + """A stand-in for the catalog tables, counting how often it is read. + + ``fingerprint`` is what the cheap check reports. ``rows_fingerprint`` is + what comes back with the rows, and is the same thing unless a test is + simulating a commit that landed between the two reads. + """ + + def __init__(self, tables: dict[str, list[str]]) -> None: + self.rows = _rows(tables) + self.fingerprint = "1" + self.rows_fingerprint: str | None = None + self.reads = 0 + + def fetch_rows(self, schema_ids: list | None = None) -> list[dict[str, str]]: + self.reads += 1 + return self.rows + + def fetch_snapshot(self) -> tuple[str, list[dict[str, str]]]: + self.reads += 1 + return self.rows_fingerprint or self.fingerprint, self.rows + + def install(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(graph_schemas, "fetch_schemas_by_ids", self.fetch_rows) + monkeypatch.setattr(graph_schemas, "fetch_all_schema_ids", lambda: []) + monkeypatch.setattr( + graph_schemas, "fetch_catalog_schema_snapshot", self.fetch_snapshot + ) + monkeypatch.setattr( + graph_schemas, + "fetch_catalog_schema_fingerprint", + lambda: self.fingerprint, + ) + + +@pytest.fixture(autouse=True) +def _fresh_snapshot() -> Any: + graph_schemas.reset_catalog_schemas() + yield + graph_schemas.reset_catalog_schemas() + + +@pytest.fixture +def catalog(monkeypatch: pytest.MonkeyPatch) -> _Catalog: + catalog = _Catalog( + { + "orders": ["order_id", "customer_id", "total"], + "customers": ["customer_id", "name"], + } + ) + catalog.install(monkeypatch) + return catalog + + +def test_unchanged_catalog_is_read_once(catalog: _Catalog) -> None: + first = graph_schemas.get_catalog_schemas() + second = graph_schemas.get_catalog_schemas() + + assert second is first + assert catalog.reads == 1 + + +def test_changed_catalog_is_rebuilt_on_the_next_call(catalog: _Catalog) -> None: + before = graph_schemas.get_catalog_schemas() + assert not before["shop"].table_exists("refunds") + + catalog.rows = catalog.rows + _rows({"refunds": ["refund_id"]}) + catalog.fingerprint = "2" + after = graph_schemas.get_catalog_schemas() + + assert after is not before + assert after["shop"].table_exists("refunds") + assert catalog.reads == 2 + + +def test_map_is_filed_under_the_fingerprint_of_its_own_rows(catalog: _Catalog) -> None: + """A commit between the check and the read must not mislabel the map. + + The check sees the catalog without ``refunds``; by the time the rows are + read, ``refunds`` has been committed. Filed under the fingerprint from the + check, that map would be served again the moment ``refunds`` was dropped + and the catalog's fingerprint returned to the earlier value -- validating + SQL against a table that no longer exists. + """ + without_refunds = catalog.rows + catalog.fingerprint = "without-refunds" + catalog.rows = without_refunds + _rows({"refunds": ["refund_id"]}) + catalog.rows_fingerprint = "with-refunds" + + assert graph_schemas.get_catalog_schemas()["shop"].table_exists("refunds") + + # refunds is dropped again: the catalog is back to the state the first + # check saw, and reports the same fingerprint it did then. + catalog.rows = without_refunds + catalog.rows_fingerprint = None + + assert not graph_schemas.get_catalog_schemas()["shop"].table_exists("refunds") + + +def test_unreadable_fingerprint_builds_fresh_every_time( + catalog: _Catalog, monkeypatch: pytest.MonkeyPatch +) -> None: + def unavailable() -> str: + raise RuntimeError("function hashtextextended does not exist") + + monkeypatch.setattr(graph_schemas, "fetch_catalog_schema_fingerprint", unavailable) + + first = graph_schemas.get_catalog_schemas() + second = graph_schemas.get_catalog_schemas() + + assert first["shop"].table_exists("orders") + assert second is not first + assert catalog.reads == 2 + + +def test_a_failed_build_is_not_kept( + catalog: _Catalog, monkeypatch: pytest.MonkeyPatch +) -> None: + """An empty catalog raises, as it always did, and leaves nothing behind.""" + populated = catalog.rows + catalog.rows = [] + with pytest.raises(KeyError): + graph_schemas.get_catalog_schemas() + + # Same fingerprint on purpose: had the failure been cached, this would + # return whatever was stored instead of building. + catalog.rows = populated + assert graph_schemas.get_catalog_schemas()["shop"].table_exists("orders") + + +_QUERIES = [ + "SELECT name FROM customers", + "SELECT * FROM shop.orders", + "SELECT o.total, c.name FROM orders o JOIN customers c " + "ON o.customer_id = c.customer_id WHERE total > 10", + "SELECT customer_id FROM orders JOIN customers USING (customer_id)", + "WITH big AS (SELECT customer_id, total FROM orders WHERE total > 100) " + "SELECT name, total FROM big JOIN customers " + "ON big.customer_id = customers.customer_id", + "SELECT order_id FROM orders UNION ALL SELECT customer_id FROM customers", + "SELECT missing_column FROM orders, customers WHERE name = 'x'", + "SELECT 1 FROM not_in_the_catalog", +] + + +def _extraction(sql: str, schemas: dict, dialect: str) -> Any: + result = extract_tables_and_columns( + sql=sql, dialects=[dialect], all_schemas=schemas + ) + return ( + {name: (sorted(m.columns), m.schema_name) for name, m in result.tables.items()}, + result.ast_node_count, + result.joins, + result.unions, + ) + + +@pytest.mark.parametrize("dialect", ["postgres", "snowflake", "databricks"]) +@pytest.mark.parametrize("sql", _QUERIES) +def test_shared_map_extracts_what_a_fresh_map_does( + catalog: _Catalog, sql: str, dialect: str +) -> None: + """Twice against the shared map, so the second run reads its caches.""" + fresh = graph_schemas.get_schemas_by_ids([]) + shared = graph_schemas.get_catalog_schemas() + assert isinstance(shared, SchemaCatalog) + + expected = _extraction(sql, fresh, dialect) + + assert _extraction(sql, shared, dialect) == expected + assert _extraction(sql, shared, dialect) == expected + + +def test_shared_map_resolves_the_same_catalog_ids(catalog: _Catalog) -> None: + sql = "SELECT o.total, c.name FROM orders o JOIN customers c ON o.customer_id = c.customer_id" + fresh = parse_query_single( + sql=sql, schemas=graph_schemas.get_schemas_by_ids([]), dialects=["postgres"] + ) + shared_schemas = graph_schemas.get_catalog_schemas() + + for _ in range(2): + with graph_schemas.catalog_schemas_lock: + shared = parse_query_single( + sql=sql, schemas=shared_schemas, dialects=["postgres"] + ) + assert sorted(shared.get_tables_ids()) == sorted(fresh.get_tables_ids()) + assert sorted(shared.get_column_ids()) == sorted(fresh.get_column_ids()) + + +def test_a_parse_does_not_see_nodes_an_earlier_parse_built( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The same SQL must report the same columns on the first parse and the tenth. + + A column stored with its quotes fails the lookup that builds its node and + is left out, but the half-built node stays behind; asked again, the schema + answers from it and the column appears. With one map shared by every + validation, what a question was told would depend on the questions before. + """ + catalog = _Catalog({"orders": ["order_id", '"Total"']}) + catalog.install(monkeypatch) + sql = "SELECT order_id, total FROM orders" + + def parsed_columns(schemas: dict) -> list[str]: + with graph_schemas.catalog_schemas_lock: + graph_schemas.forget_parsed_nodes(schemas) + query = parse_query_single(sql=sql, schemas=schemas, dialects=["postgres"]) + return sorted(query.get_column_ids()) + + fresh = parsed_columns(graph_schemas.get_schemas_by_ids([])) + assert len(fresh) == 1, "the quoted column is skipped by a fresh schema" + + shared = graph_schemas.get_catalog_schemas() + assert [parsed_columns(shared) for _ in range(3)] == [fresh] * 3 + + +def test_forgetting_nodes_leaves_a_one_off_map_alone(catalog: _Catalog) -> None: + one_off = graph_schemas.get_schemas_by_ids([]) + node = one_off["shop"].get_table_node("orders") + + graph_schemas.forget_parsed_nodes(one_off) + + assert one_off["shop"].get_table_node("orders") is node + + +def test_column_index_follows_a_replaced_frame(catalog: _Catalog) -> None: + """The per-table index is tied to the frame it was read from.""" + schema = graph_schemas.get_schemas_by_ids([])["shop"] + assert schema.column_names_by_table()["customers"] == ("customer_id", "name") + + schema.columns_df = schema.columns_df[ + schema.columns_df["column_name"] != "name" + ].copy() + + assert schema.column_names_by_table()["customers"] == ("customer_id",) + + +def test_reused_sqlglot_schema_does_not_grow_with_parses(catalog: _Catalog) -> None: + """A worker parses for days; per-query lookups must not accumulate. + + Measured over every dict sqlglot keeps on the schema except the catalog + mapping itself, so a later sqlglot that renames or adds a memo table fails + here instead of leaking quietly. + """ + shared = graph_schemas.get_catalog_schemas() + + def parse(index: int) -> None: + sql = ( + f"SELECT o{index}.total, c{index}.name FROM orders o{index} " + f"JOIN customers c{index} ON o{index}.customer_id = c{index}.customer_id" + ) + with graph_schemas.catalog_schemas_lock: + extract_tables_and_columns( + sql=sql, dialects=["postgres"], all_schemas=shared + ) + + def retained() -> int: + schema = shared._qualify_schemas["postgres"] + return sum( + len(value) + for name, value in vars(schema).items() + if isinstance(value, dict) and name not in ("mapping", "udf_mapping") + ) + + for index in range(5): + parse(index) + after_a_few = retained() + for index in range(5, 300): + parse(index) + + assert retained() == after_a_few diff --git a/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py b/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py index ce78bd7df..5c3932d78 100644 --- a/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py +++ b/auto_ontology/retrieval/text_to_sql/agents/sql_parse_validation.py @@ -41,8 +41,9 @@ ) from auto_ontology.retrieval.data_access.custom_analyses import get_custom_analyses_ids from auto_ontology.retrieval.data_access.graph_schemas import ( - fetch_all_schema_ids, - get_schemas_by_ids, + catalog_schemas_lock, + forget_parsed_nodes, + get_catalog_schemas, ) from auto_ontology.retrieval.text_to_sql.formatters_util import ( format_semantic_context, @@ -796,8 +797,9 @@ def execute(self, state: AgentState) -> Dict[str, Any]: connector = resolve_connector_from_tables( path_state.get("relevant_tables"), connectors ) - schemas_ids = fetch_all_schema_ids() - schemas = get_schemas_by_ids(schemas_ids) + # The whole catalog, shared across attempts and questions and rebuilt + # only when the catalog itself changes -- see get_catalog_schemas. + schemas = get_catalog_schemas() degenerate_dialect = getattr(connector, "dialect", None) # Every constructed/reconstructed SQL passes through this node before @@ -860,9 +862,14 @@ def execute(self, state: AgentState) -> Dict[str, Any]: "path_state": path_state, } - validation_result = self._sql_parse_validation( - schemas, response.sql_code, dialects - ) + # The schema map is shared process-wide and builds nodes as it is + # parsed against, so only one parse touches it at a time, and each + # starts from a map no earlier parse has left its nodes on. + with catalog_schemas_lock: + forget_parsed_nodes(schemas) + validation_result = self._sql_parse_validation( + schemas, response.sql_code, dialects + ) if validation_result.get("error"): error_msg = validation_result["error"] diff --git a/auto_ontology/retrieval/text_to_sql/main.py b/auto_ontology/retrieval/text_to_sql/main.py index 6fd147cf4..58eb81ab8 100644 --- a/auto_ontology/retrieval/text_to_sql/main.py +++ b/auto_ontology/retrieval/text_to_sql/main.py @@ -135,6 +135,47 @@ def __init__( ) +class _NodeTimings: + """Where one run's wall time went, node by node. + + Nodes run strictly one after another, so the time between two consecutive + node completions belongs to the later node. That needs nothing from the + nodes themselves and holds for both ways the graph is streamed. + + The answer to "why was that question slow" is otherwise only recoverable + by lining up log timestamps by hand, and it differs per question: one + spends its time generating SQL, another in four rounds of reconstruction, + another waiting on the database. + """ + + def __init__(self) -> None: + self._started = time.perf_counter() + self._last = self._started + self._seconds: dict[str, float] = {} + self._visits: dict[str, int] = {} + + def node_finished(self, node_name: str) -> None: + now = time.perf_counter() + if node_name not in _TRANSPARENT_NODES: + self._seconds[node_name] = ( + self._seconds.get(node_name, 0.0) + now - self._last + ) + self._visits[node_name] = self._visits.get(node_name, 0) + 1 + self._last = now + + def summary(self) -> str: + """Slowest node first, e.g. ``total 41.3s | reconstruct_sql 19.8s (x3) | ...``.""" + parts = [f"total {time.perf_counter() - self._started:.1f}s"] + for node_name, seconds in sorted( + self._seconds.items(), key=lambda item: item[1], reverse=True + ): + visits = self._visits[node_name] + parts.append( + f"{node_name} {seconds:.1f}s" + (f" (x{visits})" if visits > 1 else "") + ) + return " | ".join(parts) + + def _build_state(payload: TextToSQLPayload) -> AgentState: custom_prompts = payload.get("custom_prompts", "") acronyms = payload.get("acronyms", []) @@ -441,6 +482,7 @@ def stream_agent_response( streamed_sql: str | None = None last_node: str | None = None + timings = _NodeTimings() try: while True: final_state = dict(state) @@ -471,6 +513,7 @@ def stream_agent_response( logger.info("--- AGENT STEP ---") for node_name, node_output in chunk.items(): + timings.node_finished(node_name) if node_name in _TRANSPARENT_NODES: _merge_node_output(final_state, node_output) continue @@ -532,10 +575,12 @@ def stream_agent_response( answer["thoughts"] = thoughts_summary elapsed = time.perf_counter() - t0 logger.debug("Final answer (%.2fs):\n%s", elapsed, answer) + logger.info("Text-to-SQL timing: %s", timings.summary()) yield {"type": "result", "answer": answer} except Exception as exc: logger.exception("Error during agent stream") + logger.info("Text-to-SQL timing (failed run): %s", timings.summary()) yield { "type": "error", "message": f"Agent failed after {last_node or 'graph_start'}: {exc}", @@ -564,6 +609,7 @@ def get_agent_response_with_state(payload: TextToSQLPayload) -> dict: # Required by auto_ontology/retrieval/interactive/coordinator.py to persist path_state across turns/phases. state = _build_state(payload) final_state = dict(state) + timings = _NodeTimings() try: while True: @@ -571,6 +617,7 @@ def get_agent_response_with_state(payload: TextToSQLPayload) -> dict: terminal_node: str | None = None for step in app.stream(state, config={"recursion_limit": 45}): for node_name, node_output in step.items(): + timings.node_finished(node_name) if node_name not in _TRANSPARENT_NODES: terminal_node = node_name _merge_node_output(final_state, node_output) @@ -583,8 +630,10 @@ def get_agent_response_with_state(payload: TextToSQLPayload) -> dict: ) state = _restore_for_full_pipeline_retry(final_state) _apply_last_deterministic_sql_fallback(final_state, terminal_node) + logger.info("Text-to-SQL timing: %s", timings.summary()) except Exception as exc: logger.exception("Error during agent stream in get_agent_response_with_state") + logger.info("Text-to-SQL timing (failed run): %s", timings.summary()) # The stream may have already produced a valid, executed SQL query # (e.g. several reconstruction rounds succeeded) before a later node # raised — most commonly GraphRecursionError from an intent-validation diff --git a/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py b/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py index 6f868b16d..04e703747 100644 --- a/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py +++ b/auto_ontology/retrieval/text_to_sql/tests/test_intent_validation.py @@ -296,15 +296,10 @@ def test_non_short_answer_validation_omits_strict_projection_check( "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.invoke_with_structured_output" ) @patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_schemas_by_ids", + "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_catalog_schemas", return_value={}, ) -@patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.fetch_all_schema_ids", - return_value=[], -) def test_parse_failure_does_not_call_intent_llm( - _mock_schema_ids: MagicMock, _mock_schemas: MagicMock, mock_invoke: MagicMock, ) -> None: @@ -399,15 +394,10 @@ def test_parse_result_without_catalog_table_is_invalid(_mock_parse: MagicMock) - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.invoke_with_structured_output" ) @patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_schemas_by_ids", + "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_catalog_schemas", return_value={}, ) -@patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.fetch_all_schema_ids", - return_value=[], -) def test_retry_threshold_still_runs_intent_llm( - _mock_schema_ids: MagicMock, _mock_schemas: MagicMock, mock_invoke: MagicMock, ) -> None: @@ -447,15 +437,10 @@ def test_retry_threshold_still_runs_intent_llm( "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.invoke_with_structured_output" ) @patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_schemas_by_ids", + "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.get_catalog_schemas", return_value={}, ) -@patch( - "auto_ontology.retrieval.text_to_sql.agents.sql_parse_validation.fetch_all_schema_ids", - return_value=[], -) def test_final_full_pipeline_attempt_still_runs_intent_llm( - _mock_schema_ids: MagicMock, _mock_schemas: MagicMock, mock_invoke: MagicMock, ) -> None: diff --git a/auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py b/auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py new file mode 100644 index 000000000..e238594d6 --- /dev/null +++ b/auto_ontology/retrieval/text_to_sql/tests/test_node_timings.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A run reports where its time went, so a slow question can be explained.""" + +from __future__ import annotations + +import importlib +import logging +from types import ModuleType +from typing import Any + +import pytest + +from auto_ontology.utils import llm_invoke + + +@pytest.fixture +def main_module(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + monkeypatch.setattr(llm_invoke, "get_llm_client", lambda **_kwargs: None) + monkeypatch.setattr( + llm_invoke, "get_non_reasoning_llm_client", lambda **_kwargs: None + ) + return importlib.import_module("auto_ontology.retrieval.text_to_sql.main") + + +def _clock(monkeypatch: pytest.MonkeyPatch, main_module: ModuleType) -> list[float]: + """Replace the timer's clock with one the test advances by hand.""" + now = [100.0] + monkeypatch.setattr(main_module.time, "perf_counter", lambda: now[0]) + return now + + +def test_time_between_completions_belongs_to_the_later_node( + main_module: ModuleType, monkeypatch: pytest.MonkeyPatch +) -> None: + now = _clock(monkeypatch, main_module) + timings = main_module._NodeTimings() + + for node_name, seconds in [ + ("_entry_router", 0.25), + ("question_intent", 2.0), + ("construct_sql_from_candidates", 18.0), + ("validate_sql_query", 3.0), + ("reconstruct_sql", 9.0), + ("validate_sql_query", 4.5), + ]: + now[0] += seconds + timings.node_finished(node_name) + + assert timings.summary() == ( + "total 36.8s | construct_sql_from_candidates 18.0s | reconstruct_sql 9.0s" + " | validate_sql_query 7.5s (x2) | question_intent 2.0s" + ) + + +def test_a_streamed_run_logs_its_timing( + main_module: ModuleType, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + now = _clock(monkeypatch, main_module) + + class _App: + def stream(self, state: dict, **_kwargs: Any) -> Any: + for node_name, seconds in [ + ("question_intent", 1.5), + ("format_and_respond", 0.5), + ]: + now[0] += seconds + yield ( + "updates", + {node_name: {"path_state": {"final_response": {"response": "ok"}}}}, + ) + + monkeypatch.setattr(main_module, "app", _App()) + monkeypatch.setattr(main_module, "_build_state", lambda payload: {"path_state": {}}) + + with caplog.at_level(logging.INFO, logger=main_module.__name__): + events = list(main_module.stream_agent_response({"question": "q"})) + + assert events[-1] == {"type": "result", "answer": {"response": "ok"}} + assert ( + "Text-to-SQL timing: total 2.0s | question_intent 1.5s | format_and_respond 0.5s" + in caplog.text + ) diff --git a/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py b/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py index 6dc3db7fc..4a35c3658 100644 --- a/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py +++ b/auto_ontology/retrieval/text_to_sql/tests/test_numeric_types.py @@ -428,8 +428,7 @@ def _validate( ) connector = SimpleNamespace(dialect=dialect) - monkeypatch.setattr(f"{_VALIDATION}.fetch_all_schema_ids", lambda: []) - monkeypatch.setattr(f"{_VALIDATION}.get_schemas_by_ids", lambda ids: {}) + monkeypatch.setattr(f"{_VALIDATION}.get_catalog_schemas", lambda: {}) monkeypatch.setattr( f"{_VALIDATION}.resolve_connector_from_tables", lambda tables, conns: connector ) diff --git a/auto_ontology/server/chat/tests/test_worker.py b/auto_ontology/server/chat/tests/test_worker.py index b283cf77a..85c5948cc 100644 --- a/auto_ontology/server/chat/tests/test_worker.py +++ b/auto_ontology/server/chat/tests/test_worker.py @@ -33,6 +33,8 @@ def _run_loop( connector_batches: list[list[str]], *, evidence: str | None = None, + out_q: _FakeQueue | None = None, + prewarm: Any = lambda *retrievers: None, ) -> dict: """Drive one ASK through ``_worker_loop`` and return the agent payload it built. @@ -79,9 +81,11 @@ def fake_stream(payload: dict) -> Any: monkeypatch.setattr( "auto_ontology.retrieval.text_to_sql.main.stream_agent_response", fake_stream ) + # Warming opens real connections; these tests are about the request loop. + monkeypatch.setattr(worker_module, "_prewarm", prewarm) in_q = _FakeQueue([(worker_module._MSG_ASK, ("q", None, None, None, [], evidence))]) - worker_module._worker_loop(in_q, _FakeQueue()) + worker_module._worker_loop(in_q, out_q or _FakeQueue()) return captured @@ -122,3 +126,91 @@ def test_worker_forwards_evidence_without_changing_question( assert payload["question"] == "q" assert payload["processing_question"] == "q" assert payload["evidence"] == "status means accounts.status" + + +class _Vdb: + def __init__(self, error: Exception | None = None) -> None: + self.error = error + self.warmed = 0 + + def warm(self) -> bool: + self.warmed += 1 + if self.error is not None: + raise self.error + return True + + +class _Retriever: + def __init__(self, vdb: _Vdb) -> None: + self.vdb = vdb + + +def _stub_prewarm_targets( + monkeypatch: MonkeyPatch, *, store_error: Exception | None = None +) -> list[str]: + calls: list[str] = [] + + class _Store: + def query_read(self, statement: str) -> list: + calls.append(statement) + if store_error is not None: + raise store_error + return [] + + def empty_catalog() -> dict: + calls.append("schema map") + raise KeyError("database_name") + + monkeypatch.setattr("auto_ontology.dal.session.store", lambda: _Store()) + monkeypatch.setattr( + "auto_ontology.retrieval.data_access.graph_schemas.get_catalog_schemas", + empty_catalog, + ) + return calls + + +def test_prewarm_opens_every_store_and_survives_a_failing_one( + monkeypatch: MonkeyPatch, +) -> None: + """One store being down must not leave the others cold, or stop the worker.""" + calls = _stub_prewarm_targets(monkeypatch) + broken = _Vdb(RuntimeError("connection refused")) + healthy = _Vdb() + + worker_module._prewarm(_Retriever(broken), _Retriever(healthy)) + + assert calls == ["SELECT 1", "schema map"] + assert (broken.warmed, healthy.warmed) == (1, 1) + + +def test_prewarm_stops_when_the_database_is_unreachable( + monkeypatch: MonkeyPatch, +) -> None: + """The vector stores share that database and connect with no timeout.""" + calls = _stub_prewarm_targets( + monkeypatch, store_error=RuntimeError("connection timeout expired") + ) + vdb = _Vdb() + + worker_module._prewarm(_Retriever(vdb)) + + assert calls == ["SELECT 1"] + assert vdb.warmed == 0 + + +def test_worker_reports_ready_before_it_warms(monkeypatch: MonkeyPatch) -> None: + """The pool waits on READY with no liveness check; warming must not precede it.""" + out_q = _FakeQueue() + ready_when_warming: list[bool] = [] + + def record(*_retrievers: object) -> None: + ready_when_warming.append((worker_module._TAG_READY, None) in out_q.puts) + + _run_loop( + monkeypatch, + connector_batches=[["boot-connector"]], + out_q=out_q, + prewarm=record, + ) + + assert ready_when_warming == [True] diff --git a/auto_ontology/server/chat/worker.py b/auto_ontology/server/chat/worker.py index 2dcb1198b..ce3c61a4a 100644 --- a/auto_ontology/server/chat/worker.py +++ b/auto_ontology/server/chat/worker.py @@ -56,6 +56,49 @@ _TERMINATE_GRACE_S = 1.0 +def _prewarm(*retrievers: Any) -> None: + """Pay the first question's one-time costs before there is a question. + + Importing the agent and building the singletons leaves several things + unopened: the catalog store's connection pool, each vector store's engine + and table binding, and the catalog schema map SQL validation parses + against. Each is created on first use, so without this the first question + a worker serves is slower than every later one -- and a worker is replaced + on every cancel, so "first" recurs. + + Best-effort and read-only. A step that fails is skipped and happens lazily + at its usual place instead. An empty catalog, for one, has no schema map + to build yet. + + The catalog store goes first and decides whether anything else is tried. + It is the one connection here with a connect timeout, and the vector + stores live in the same database: if it cannot be reached there is nothing + to warm, and trying would only wait on a connection that is not coming. + """ + from auto_ontology.dal.session import store + from auto_ontology.retrieval.data_access.graph_schemas import get_catalog_schemas + + try: + store().query_read("SELECT 1") + except Exception as exc: # noqa: BLE001 — warming is optional + logger.info( + "Prewarm skipped: the catalog store is not reachable (%s: %s)", + type(exc).__name__, + exc, + ) + return + + steps: list[tuple[str, Any]] = [("catalog schema map", get_catalog_schemas)] + for retriever in retrievers: + steps.append(("vector store", lambda r=retriever: r.vdb.warm())) + + for name, step in steps: + try: + step() + except Exception as exc: # noqa: BLE001 — warming is optional + logger.info("Prewarm skipped the %s: %s: %s", name, type(exc).__name__, exc) + + def _worker_loop( in_q: "mp.Queue[tuple[str, Any]]", out_q: "mp.Queue[tuple[str, Any]]", @@ -123,6 +166,13 @@ def _worker_loop( except Exception: # noqa: BLE001 return + # After READY, not before. The pool waits for READY with no timeout and no + # liveness check, so a worker that died while warming -- building the + # schema map for a very large catalog is the heaviest thing it does at + # start -- would leave that wait blocked for good. Once READY is out, a + # death here is seen the way a death mid-question is: by events(). + _prewarm(data_retriever, semantic_retriever) + while True: try: message = in_q.get() diff --git a/auto_ontology/utils/llm_invoke.py b/auto_ontology/utils/llm_invoke.py index bc1694f21..b7f9ad1f3 100644 --- a/auto_ontology/utils/llm_invoke.py +++ b/auto_ontology/utils/llm_invoke.py @@ -11,7 +11,8 @@ import threading import time import weakref -from typing import Iterator, Type, TypeVar +from http.cookiejar import DefaultCookiePolicy +from typing import Any, Iterator, Type, TypeVar import requests as _requests from langchain_core.language_models import BaseChatModel @@ -183,16 +184,116 @@ def _is_retryable(e: Exception) -> bool: ) +# How long a connection may sit unused in a session before it is dropped rather +# than reused. A server or load balancer closes idle connections on its own +# schedule, and a request written to one it has just closed fails instead of +# being answered. Five seconds is the window httpx uses, so it is the one the +# ChatOpenAI path has always run with; it comfortably covers the gap between +# consecutive model calls in one question, which is what reuse is for. +LLM_KEEPALIVE_IDLE_S = float(os.environ.get("LLM_KEEPALIVE_IDLE_S", "5")) + +# A request written to a connection the far end already closed fails within a +# network round trip. A failure later than this is not that: the request was +# delivered and something went wrong while it was being answered. +_STALE_CONNECTION_WINDOW_S = 1.0 + + class _TimeoutSession(_requests.Session): - """requests.Session that enforces a default timeout on every request.""" + """requests.Session with a default timeout, safe to keep and reuse. + + ``ChatNVIDIA`` asks for a session before every request. Handing it a new + one each time means a TCP and TLS handshake per model call -- measured at + 100 ms against integrate.api.nvidia.com and 250 ms against + inference-api.nvidia.com -- which a question pays seven to twenty times. + Reusing one keeps the connection open between calls. + + Reuse brings one failure a fresh session cannot have: the far end closes a + connection just as a request is written to it. Three things keep that from + reaching the caller, and keep everything else as it was: + + * Connections idle for longer than ``max_idle`` are closed before the next + request, so one the far end has probably dropped is not written to. + * A request on a reused session that fails with a connection error within + a second is sent once more on a new connection. urllib3 already + discards a pooled connection the far end has visibly closed; this covers + the far end closing it in the moment between that check and the write. + The window is a heuristic, not proof the request was never delivered: + an endpoint that reads a request and drops the connection within a + second gets it twice. That is the deliberate side to err on. Without the + resend, the lost race reaches the caller as an error its retry policy + does not retry, and the call falls back to another model or returns + nothing -- a worse answer, where the resend costs one generation. A + failure later than a second is left to the caller exactly as a fresh + session's would be, and timeouts are never resent, whichever exception + they arrive as. + * No cookie is ever stored. A one-shot session never sent any, and one set + by a load balancer would otherwise pin every later call to one backend. + """ - def __init__(self, timeout: float = LLM_INVOKE_TIMEOUT_S, **kwargs): + def __init__( + self, + timeout: float = LLM_INVOKE_TIMEOUT_S, + *, + max_idle: float = LLM_KEEPALIVE_IDLE_S, + **kwargs: Any, + ) -> None: super().__init__(**kwargs) self._default_timeout = timeout - - def request(self, method, url, **kwargs): + self._max_idle = max_idle + self._clock = time.monotonic + # When the last request finished, or None while no pooled connection + # can exist (never used, or the pool was just emptied). + self._last_used: float | None = None + self.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[])) + + def request(self, method: Any, url: Any, **kwargs: Any) -> _requests.Response: kwargs.setdefault("timeout", self._default_timeout) - return super().request(method, url, **kwargs) + started = self._clock() + reused = self._last_used is not None + if reused and started - self._last_used > self._max_idle: + self.close() + reused = False + self._last_used = None + try: + response = super().request(method, url, **kwargs) + except _requests.exceptions.ConnectionError as exc: + stale = ( + reused + and not isinstance(exc, _requests.exceptions.Timeout) + and self._clock() - started <= _STALE_CONNECTION_WINDOW_S + ) + if not stale: + raise + logger.info( + "LLM connection was closed while idle (%s); retrying once on a " + "new connection", + type(exc).__name__, + ) + self.close() + response = super().request(method, url, **kwargs) + self._last_used = self._clock() + return response + + +class _ThreadSessions: + """One reusable :class:`_TimeoutSession` per thread, for one client. + + A ``requests.Session`` is not documented as safe to share between threads, + and nothing here needs it to be: a question's model calls run one after + another on a single thread, so a session per thread reuses the connection + for exactly the calls that are close together, and concurrent questions + never touch each other's. + """ + + def __init__(self, timeout: float) -> None: + self._timeout = timeout + self._local = threading.local() + + def __call__(self) -> _TimeoutSession: + session = getattr(self._local, "session", None) + if session is None: + session = self._local.session = _TimeoutSession(self._timeout) + return session T = TypeVar("T", bound=BaseModel) @@ -249,8 +350,14 @@ def _build_client( base_url=base_url, temperature=temperature, max_tokens=max_tokens, + # Without this the client passes its own default of 60 seconds on + # every request, which silently overrides the session default below: + # LLM_INVOKE_TIMEOUT_S was never the limit in force on this path, and a + # generation that needed longer than a minute was abandoned, retried + # twice the same way, and reported as no answer. + timeout=LLM_INVOKE_TIMEOUT_S, ) - client._client.get_session_fn = lambda: _TimeoutSession(LLM_INVOKE_TIMEOUT_S) + client._client.get_session_fn = _ThreadSessions(LLM_INVOKE_TIMEOUT_S) return client @@ -319,6 +426,10 @@ def get_non_reasoning_llm_client( _NON_REASONING_STRONG_REFS: dict[int, object] = {} _REASONING_FALLBACK_CLIENT: BaseChatModel | None = None _REASONING_FALLBACK_LOCK = threading.Lock() +# Per thread: the default non-reasoning client and the factory that built it. +# See ``_get_default_non_reasoning_client`` for why it is per thread and why +# the factory is remembered. +_DEFAULT_NON_REASONING = threading.local() # How many ``bound`` links to follow. Real ``.bind()`` chains are a level or two # deep; the cap stops the walk on objects whose ``bound`` never ends — a @@ -363,6 +474,32 @@ def _get_reasoning_fallback_client() -> BaseChatModel: return _REASONING_FALLBACK_CLIENT +def _get_default_non_reasoning_client() -> BaseChatModel: + """This thread's non-reasoning client with default settings, built once. + + ``safe_invoke_text_nr`` and ``safe_invoke_structured_nr`` used to build a + client per call. A client owns its connection pool, so each of those calls + opened a new connection, did one request on it, and threw it away. A + failure to build is not remembered: the next call tries again. + + One per thread rather than one for the process, because these calls never + shared a client before and ``ChatNVIDIA`` is not built to be shared: it + stores the outgoing request on the client object and reads it back to send + it, so two threads on one client can each send the other's prompt. The + calls that benefit from reuse are consecutive ones on the same thread. + + The client is kept together with the factory that produced it and is + rebuilt if ``get_non_reasoning_llm_client`` is no longer that function. + Several test modules replace the factory; without this, a client one of + them built would outlive the patch and be handed to an unrelated caller. + """ + factory = get_non_reasoning_llm_client + cached = getattr(_DEFAULT_NON_REASONING, "client", None) + if cached is None or cached[0] is not factory: + cached = _DEFAULT_NON_REASONING.client = (factory, factory()) + return cached[1] + + def _reasoning_fallback_client(llm: object) -> BaseChatModel | None: """The reasoning client when *llm* is non-reasoning, else ``None``. @@ -691,7 +828,7 @@ def safe_invoke_text_nr(prompt: str) -> str: that single call when the non-reasoning endpoint stays down. """ try: - client = get_non_reasoning_llm_client() + client = _get_default_non_reasoning_client() except Exception as exc: _log_non_reasoning_fallback(exc) client = _get_reasoning_fallback_client() @@ -708,7 +845,7 @@ def safe_invoke_structured_nr( model for that single call. Returns ``None`` when both models fail. """ try: - client = get_non_reasoning_llm_client() + client = _get_default_non_reasoning_client() except Exception as exc: schema_name = getattr(schema, "__name__", str(schema)) _log_non_reasoning_fallback(exc, schema_name) diff --git a/auto_ontology/utils/retriever.py b/auto_ontology/utils/retriever.py index cc04b16bd..61b2b775b 100644 --- a/auto_ontology/utils/retriever.py +++ b/auto_ontology/utils/retriever.py @@ -7,6 +7,11 @@ from __future__ import annotations import logging +import os +import threading +from array import array +from collections import OrderedDict +from concurrent.futures import Future from typing import Any from auto_ontology.utils.embedding import EmbedParams, embed_texts, get_embed_params @@ -22,6 +27,99 @@ # endpoint costs three 60s attempts, not three times six. _QUERY_TIMEOUT_S = 60.0 +# Query vectors kept per process. One 2048-dimension vector is 16 KB as packed +# doubles, so the default holds about 16 MB. Zero turns the cache off. +_QUERY_CACHE_SIZE = int(os.environ.get("AUTO_ONTOLOGY_QUERY_EMBED_CACHE_SIZE", "1024")) + + +class _QueryEmbeddingCache: + """Embed each distinct query string once. + + One question embeds the same few strings again and again: each entity is + searched for attributes during retrieval, for tables during preparation, + again to rank hub siblings, and once more by every reconstruction that + goes looking for a missing table; a full-pipeline retry repeats all of it. + Each of those was its own request to the embedding endpoint, on the + question's critical path whenever nothing else was running, and each one a + chance to be rate-limited into a multi-second backoff. + + The cached value is the endpoint's own answer for that exact request -- + same model, same input type, same text -- held as packed doubles, so a hit + returns bit-for-bit the vector a new request would have. Nothing is + approximated and nothing is batched: a batch is a different request and is + not guaranteed the same floats. + + Concurrent requests for a string not yet cached share one call. Retrieval + searches the full question under two labels at the same moment, and without + this both would miss and both would embed it. A failure is never stored: + every waiter gets the exception, and the next request tries again. + """ + + def __init__(self, max_entries: int) -> None: + self._max_entries = max_entries + self._lock = threading.Lock() + self._vectors: OrderedDict[tuple, array] = OrderedDict() + self._pending: dict[tuple, Future] = {} + + def get(self, key: tuple, compute: Any) -> list[float] | None: + """The vector for *key*, calling *compute* only if nobody has it yet. + + ``None`` from *compute* (a blank query) is returned and not kept. + """ + if self._max_entries <= 0: + return compute() + + with self._lock: + packed = self._vectors.get(key) + if packed is not None: + self._vectors.move_to_end(key) + return packed.tolist() + pending = self._pending.get(key) + if pending is None: + pending = self._pending[key] = Future() + owner = True + else: + owner = False + + if not owner: + vector = pending.result() + return None if vector is None else list(vector) + + try: + vector = compute() + except BaseException as exc: + with self._lock: + self._pending.pop(key, None) + pending.set_exception(exc) + raise + + # Packing can only fail for something that is not a flat list of + # numbers. That is not this cache's to judge: the vector goes on to the + # store exactly as it would have, and is simply not kept. Whatever + # happens here, the waiters are released. + packed: array | None = None + if vector is not None: + try: + packed = array("d", vector) + except (TypeError, ValueError, OverflowError): + packed = None + + with self._lock: + self._pending.pop(key, None) + if packed is not None: + self._vectors[key] = packed + while len(self._vectors) > self._max_entries: + self._vectors.popitem(last=False) + pending.set_result(vector) + return vector + + def clear(self) -> None: + with self._lock: + self._vectors.clear() + + +_query_embeddings = _QueryEmbeddingCache(_QUERY_CACHE_SIZE) + class Retriever: """Embed a query and run a k-NN search against one :class:`PostgresVDB`. @@ -54,13 +152,7 @@ def query( embedded, with the endpoint's HTTP status in the message, so callers can tell a transient failure from an empty result. """ - vector = embed_texts( - [text], - self.embed_params, - "query", - timeout=_QUERY_TIMEOUT_S, - retries=0, - )[0] + vector = self._embed_query(text) if vector is None: raise ValueError("Cannot search for a blank query.") where = (vdb_kwargs or {}).get("where") @@ -74,6 +166,38 @@ def query( for hit in hits[:top_k] ] + def _embed_query(self, text: str) -> list[float] | None: + """The query vector for *text*, embedded at most once per process. + + The key is everything that decides the request ``embed_texts`` sends: + where it goes, which model answers, the modality, and the text as it + is sent (stripped). See :class:`_QueryEmbeddingCache`. + + The embedding function itself is part of the key as well. In the + running service it is always the same function; a test that swaps in + its own embedder gets its own entries instead of vectors another + embedder produced for the same string. + """ + params = self.embed_params + embed = embed_texts + key = ( + embed, + params.embed_invoke_url, + params.model_name, + params.embed_modality, + text.strip() if isinstance(text, str) else text, + ) + return _query_embeddings.get( + key, + lambda: embed( + [text], + params, + "query", + timeout=_QUERY_TIMEOUT_S, + retries=0, + )[0], + ) + _data_retriever: Retriever | None = None _semantic_retriever: Retriever | None = None diff --git a/auto_ontology/utils/tests/test_llm_invoke.py b/auto_ontology/utils/tests/test_llm_invoke.py index 3ce7558ec..03f6e2801 100644 --- a/auto_ontology/utils/tests/test_llm_invoke.py +++ b/auto_ontology/utils/tests/test_llm_invoke.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock import pytest +import requests from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage from pydantic import BaseModel @@ -27,6 +28,7 @@ def _isolate_fallback_state(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(llm_invoke, "_NON_REASONING_IDS", set()) monkeypatch.setattr(llm_invoke, "_NON_REASONING_STRONG_REFS", {}) monkeypatch.setattr(llm_invoke, "_REASONING_FALLBACK_CLIENT", None) + monkeypatch.setattr(llm_invoke, "_DEFAULT_NON_REASONING", threading.local()) class _Model: @@ -497,3 +499,294 @@ def unavailable(**_kwargs: object) -> object: monkeypatch.setattr(llm_invoke, "get_llm_client", unavailable) assert llm_invoke.invoke_with_structured_output(primary, [], _Schema) is None + + +# --------------------------------------------------------------------------- +# Connection reuse +# --------------------------------------------------------------------------- + + +class _Endpoint: + """A local HTTP/1.1 endpoint that records which connection served each request.""" + + def __init__(self) -> None: + import http.server + + endpoint = self + self.connections: list[tuple[str, int]] = [] + self.cookies: list[str | None] = [] + # When set, the next request is read in full and recorded, and the + # connection is then closed without a response. + self.drop_next = False + + class Handler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: # noqa: N802 — http.server's name + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + endpoint.connections.append(self.client_address) + endpoint.cookies.append(self.headers.get("Cookie")) + if endpoint.drop_next: + endpoint.drop_next = False + self.close_connection = True + return + body = b"{}" + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Set-Cookie", "backend=node-7; Path=/") + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args: object) -> None: + pass + + self._server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self._server.server_address[1]}/v1/chat" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + + +@pytest.fixture +def endpoint() -> Any: + server = _Endpoint() + yield server + server.close() + + +def test_consecutive_calls_share_one_connection(endpoint: _Endpoint) -> None: + session = llm_invoke._TimeoutSession(5) + + for _ in range(3): + assert session.post(endpoint.url, json={"q": 1}).status_code == 200 + + assert len(set(endpoint.connections)) == 1 + + +def test_a_connection_left_idle_is_replaced_not_reused(endpoint: _Endpoint) -> None: + session = llm_invoke._TimeoutSession(5, max_idle=0.0) + + session.post(endpoint.url, json={}) + session.post(endpoint.url, json={}) + + assert len(set(endpoint.connections)) == 2 + + +def test_reused_session_sends_no_cookie(endpoint: _Endpoint) -> None: + """A one-shot session never carried one; a kept one must not start to.""" + session = llm_invoke._TimeoutSession(5) + + session.post(endpoint.url, json={}) + session.post(endpoint.url, json={}) + + assert endpoint.cookies == [None, None] + assert len(session.cookies) == 0 + + +class _FlakySession(llm_invoke._TimeoutSession): + """Raises the queued errors, one per send, before letting requests through.""" + + def __init__(self, errors: list[Exception], **kwargs: Any) -> None: + super().__init__(5, **kwargs) + self.errors = errors + self.sends = 0 + + def send(self, request: Any, **kwargs: Any) -> Any: + self.sends += 1 + if self.errors: + error = self.errors.pop(0) + # A callable is called at failure time, so it can move the clock. + raise error() if callable(error) else error + return super().send(request, **kwargs) + + +def test_stale_connection_is_retried_once_on_a_new_one(endpoint: _Endpoint) -> None: + session = _FlakySession([]) + session.post(endpoint.url, json={}) + session.errors.append(requests.exceptions.ConnectionError("Connection aborted.")) + + response = session.post(endpoint.url, json={}) + + assert response.status_code == 200 + assert session.sends == 3 + assert len(set(endpoint.connections)) == 2 + + +def test_a_failure_while_the_endpoint_was_answering_is_not_resent( + endpoint: _Endpoint, +) -> None: + """Seconds in, the request was delivered: resending would deliver it twice. + + This is the failure a fresh session has too, so it goes to the caller's + retry policy as it always did. requests reports a stalled response body as + a plain ConnectionError as well, which is why elapsed time decides this + and not the exception type. + """ + session = _FlakySession([]) + session.post(endpoint.url, json={}) + now = [session._clock()] + session._clock = lambda: now[0] + + def dropped_after_thirty_seconds() -> Exception: + now[0] += 30.0 + return requests.exceptions.ConnectionError("Read timed out.") + + session.errors.append(dropped_after_thirty_seconds) + + with pytest.raises(requests.exceptions.ConnectionError): + session.post(endpoint.url, json={}) + + assert session.sends == 2 + + +def test_a_request_dropped_right_after_delivery_is_sent_twice( + endpoint: _Endpoint, +) -> None: + """The documented cost of the resend: the window cannot see delivery. + + The endpoint reads the whole request and closes without answering, inside + the window, which is indistinguishable from a connection closed just + before the write. The request is delivered twice and the call succeeds, + where without the resend it would have failed outright. + """ + session = llm_invoke._TimeoutSession(5) + session.post(endpoint.url, json={}) + endpoint.drop_next = True + + response = session.post(endpoint.url, json={"q": 1}) + + assert response.status_code == 200 + assert len(endpoint.connections) == 3 + assert len(set(endpoint.connections)) == 2 + + +def test_first_request_connection_failure_is_not_retried(endpoint: _Endpoint) -> None: + """Nothing was pooled yet, so the failure is the endpoint's, not staleness.""" + session = _FlakySession([requests.exceptions.ConnectionError("refused")]) + + with pytest.raises(requests.exceptions.ConnectionError): + session.post(endpoint.url, json={}) + + assert session.sends == 1 + + +def test_timeout_on_a_reused_session_is_not_retried(endpoint: _Endpoint) -> None: + session = _FlakySession([]) + session.post(endpoint.url, json={}) + session.errors.append(requests.exceptions.ConnectTimeout("timed out")) + + with pytest.raises(requests.exceptions.ConnectTimeout): + session.post(endpoint.url, json={}) + + assert session.sends == 2 + + +def test_a_second_connection_failure_is_raised(endpoint: _Endpoint) -> None: + session = _FlakySession([]) + session.post(endpoint.url, json={}) + session.errors.extend( + [ + requests.exceptions.ConnectionError("aborted"), + requests.exceptions.ConnectionError("refused"), + ] + ) + + with pytest.raises(requests.exceptions.ConnectionError, match="refused"): + session.post(endpoint.url, json={}) + + # The failed retry left nothing pooled, so the next call starts clean. + assert session.post(endpoint.url, json={}).status_code == 200 + + +def test_each_thread_gets_its_own_session() -> None: + sessions = llm_invoke._ThreadSessions(5) + seen: list[object] = [] + + worker = threading.Thread(target=lambda: seen.append(sessions())) + worker.start() + worker.join() + + assert sessions() is sessions() + assert seen[0] is not sessions() + + +def test_nvidia_client_reuses_sessions_and_honours_the_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The client's own 60s default used to override the configured timeout.""" + monkeypatch.setattr(llm_invoke, "LLM_INVOKE_TIMEOUT_S", 137) + + client = llm_invoke._build_client( + model="nvidia/nemotron-3.5-lightning-30b-a3b", + api_key="nvapi-test", + base_url="http://127.0.0.1:9/v1", + temperature=0.0, + max_tokens=16, + ) + + assert client._client.timeout == 137 + assert client._client.get_session_fn() is client._client.get_session_fn() + + +def test_default_non_reasoning_client_is_built_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + built: list[object] = [] + + def factory(**_kwargs: object) -> object: + built.append(object()) + return built[-1] + + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", factory) + monkeypatch.setattr( + llm_invoke, + "invoke_with_structured_output", + lambda llm, _messages, _schema: llm, + ) + monkeypatch.setattr(llm_invoke, "safe_invoke_text", lambda llm, _prompt: llm) + + first = llm_invoke.safe_invoke_structured_nr([], _Schema) + second = llm_invoke.safe_invoke_structured_nr([], _Schema) + third = llm_invoke.safe_invoke_text_nr("question") + + assert first is second is third + assert len(built) == 1 + + +def test_default_non_reasoning_client_is_not_shared_between_threads( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """These calls built a client each before; they must still not share one. + + ChatNVIDIA keeps the outgoing request on the client object between + building it and sending it, so a client used from two threads can send one + thread's prompt on behalf of the other. + """ + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", lambda: object()) + monkeypatch.setattr(llm_invoke, "safe_invoke_text", lambda llm, _prompt: llm) + other: list[object] = [] + + worker = threading.Thread( + target=lambda: other.append(llm_invoke.safe_invoke_text_nr("question")) + ) + worker.start() + worker.join() + + mine = llm_invoke.safe_invoke_text_nr("question") + assert mine is llm_invoke.safe_invoke_text_nr("question") + assert other[0] is not mine + + +def test_default_non_reasoning_client_follows_a_replaced_factory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(llm_invoke, "safe_invoke_text", lambda llm, _prompt: llm) + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", lambda: "first") + assert llm_invoke.safe_invoke_text_nr("question") == "first" + + monkeypatch.setattr(llm_invoke, "get_non_reasoning_llm_client", lambda: "second") + assert llm_invoke.safe_invoke_text_nr("question") == "second" diff --git a/auto_ontology/utils/tests/test_retriever_query_cache.py b/auto_ontology/utils/tests/test_retriever_query_cache.py new file mode 100644 index 000000000..e53982e5b --- /dev/null +++ b/auto_ontology/utils/tests/test_retriever_query_cache.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A query string is embedded once, and what comes back is exactly that vector. + +One question searches for the same entity several times across retrieval, +preparation and reconstruction. Each search used to be its own request to the +embedding endpoint. Reusing the answer is only acceptable if it is the same +answer, and if a failed or blank request is never mistaken for one. +""" + +from __future__ import annotations + +import threading +from concurrent.futures import Future +from typing import Any + +import pytest + +from auto_ontology.utils import retriever as retriever_mod +from auto_ontology.utils.embedding import EmbedParams +from auto_ontology.utils.retriever import Retriever, _QueryEmbeddingCache + + +class _Store: + """Records the vectors it is searched with.""" + + def __init__(self) -> None: + self.searched: list[list[float]] = [] + + def retrieval(self, queries: list, top_k: int = 10, **kwargs: Any) -> list: + self.searched.extend(queries) + return [[] for _ in queries] + + +class _Embedder: + """Stands in for ``embed_texts``, counting requests per text.""" + + def __init__(self) -> None: + self.requests: list[str] = [] + self.fail_with: Exception | None = None + self.gate: threading.Event | None = None + + def __call__( + self, texts: list[str], params: Any, input_type: str, **kw: Any + ) -> list: + self.requests.extend(texts) + if self.gate is not None: + assert self.gate.wait(5) + if self.fail_with is not None: + raise self.fail_with + if not texts[0].strip(): + return [None] + # Values that do not survive float32, so a lossy cache would show. + return [[len(texts[0]) / 3.0, 0.1 + 0.2, 1e-17]] + + +@pytest.fixture +def embedder(monkeypatch: pytest.MonkeyPatch) -> _Embedder: + embedder = _Embedder() + monkeypatch.setattr(retriever_mod, "embed_texts", embedder) + monkeypatch.setattr(retriever_mod, "_query_embeddings", _QueryEmbeddingCache(8)) + return embedder + + +def _retriever(model: str = "model-a") -> tuple[Retriever, _Store]: + store = _Store() + return Retriever(store, EmbedParams(model, "http://embed.test/v1")), store # type: ignore[arg-type] + + +def test_repeated_query_is_embedded_once(embedder: _Embedder) -> None: + retriever, store = _retriever() + + retriever.query("total revenue") + retriever.query("total revenue", top_k=3) + retriever.query(" total revenue ") + + assert embedder.requests == ["total revenue"] + assert store.searched[0] == store.searched[1] == store.searched[2] + + +def test_cached_vector_is_bit_identical(embedder: _Embedder) -> None: + retriever, store = _retriever() + + retriever.query("margin") + retriever.query("margin") + + first, second = store.searched + assert first == [len("margin") / 3.0, 0.1 + 0.2, 1e-17] + assert [value.hex() for value in second] == [value.hex() for value in first] + assert isinstance(second, list) + + +def test_different_text_or_model_is_a_different_request(embedder: _Embedder) -> None: + retriever_a, _ = _retriever("model-a") + retriever_b, _ = _retriever("model-b") + + retriever_a.query("orders") + retriever_a.query("Orders") + retriever_b.query("orders") + + assert embedder.requests == ["orders", "Orders", "orders"] + + +def test_failure_is_not_cached(embedder: _Embedder) -> None: + retriever, _ = _retriever() + embedder.fail_with = RuntimeError("502 Bad Gateway") + + with pytest.raises(RuntimeError, match="502"): + retriever.query("orders") + + embedder.fail_with = None + retriever.query("orders") + + assert embedder.requests == ["orders", "orders"] + + +def test_blank_query_still_raises_and_is_not_cached(embedder: _Embedder) -> None: + retriever, _ = _retriever() + + for _ in range(2): + with pytest.raises(ValueError, match="blank query"): + retriever.query(" ") + + assert embedder.requests == [" ", " "] + + +def test_concurrent_requests_for_one_string_share_a_call(embedder: _Embedder) -> None: + """Retrieval searches the question under two labels at the same moment.""" + retriever, store = _retriever() + embedder.gate = threading.Event() + threads = [ + threading.Thread(target=retriever.query, args=("how many orders",)) + for _ in range(4) + ] + for thread in threads: + thread.start() + while not embedder.requests: + pass + embedder.gate.set() + for thread in threads: + thread.join(5) + + assert embedder.requests == ["how many orders"] + assert len(store.searched) == 4 + assert all(vector == store.searched[0] for vector in store.searched) + + +def test_concurrent_waiters_see_the_owners_failure(embedder: _Embedder) -> None: + retriever, _ = _retriever() + embedder.gate = threading.Event() + embedder.fail_with = RuntimeError("503 Service Unavailable") + errors: list[BaseException] = [] + + def search() -> None: + try: + retriever.query("orders") + except BaseException as exc: # noqa: BLE001 — recorded for the assertion + errors.append(exc) + + threads = [threading.Thread(target=search) for _ in range(3)] + for thread in threads: + thread.start() + while not embedder.requests: + pass + embedder.gate.set() + for thread in threads: + thread.join(5) + + assert len(errors) == 3 + assert all("503" in str(error) for error in errors) + + +class _CountingFuture(Future): + """A Future that counts the threads blocked on its result.""" + + def __init__(self) -> None: + super().__init__() + self.waiting = 0 + self._count_lock = threading.Lock() + + def result(self, timeout: float | None = None) -> Any: + with self._count_lock: + self.waiting += 1 + return super().result(timeout) + + +def test_a_vector_that_cannot_be_packed_is_passed_on_and_strands_nobody( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Whatever the endpoint returned still reaches the store, and every waiter.""" + futures: list[_CountingFuture] = [] + + def make_future() -> _CountingFuture: + futures.append(_CountingFuture()) + return futures[-1] + + monkeypatch.setattr(retriever_mod, "Future", make_future) + cache = _QueryEmbeddingCache(8) + gate = threading.Event() + odd = [[0.1, 0.2]] # nested: not a flat list of numbers + calls: list[int] = [] + + def compute() -> list: + calls.append(1) + assert gate.wait(5) + return odd + + results: list[object] = [] + threads = [ + threading.Thread(target=lambda: results.append(cache.get(("k",), compute))) + for _ in range(3) + ] + for thread in threads: + thread.start() + # Release the owner only once the other two are blocked on its result, so + # none of them can arrive late and become an owner itself. + deadline = threading.Event() + for _ in range(500): + if futures and futures[0].waiting == 2: + break + deadline.wait(0.01) + assert futures and futures[0].waiting == 2 + gate.set() + for thread in threads: + thread.join(5) + + assert not any(thread.is_alive() for thread in threads) + assert results == [odd, odd, odd] + assert len(calls) == 1 + # Not kept, so the next request asks again instead of reusing it. + assert cache.get(("k",), compute) == odd + assert len(calls) == 2 + + +def test_oldest_entry_is_evicted_first(embedder: _Embedder) -> None: + retriever, _ = _retriever() + retriever_mod._query_embeddings = _QueryEmbeddingCache(2) + + retriever.query("a") + retriever.query("b") + retriever.query("a") # a is now the most recently used + retriever.query("c") # evicts b + retriever.query("a") + retriever.query("b") + + assert embedder.requests == ["a", "b", "c", "b"] + + +def test_size_zero_disables_the_cache(embedder: _Embedder) -> None: + retriever, _ = _retriever() + retriever_mod._query_embeddings = _QueryEmbeddingCache(0) + + retriever.query("orders") + retriever.query("orders") + + assert embedder.requests == ["orders", "orders"] diff --git a/auto_ontology/vdb/postgres.py b/auto_ontology/vdb/postgres.py index 68f48c083..7b588b773 100644 --- a/auto_ontology/vdb/postgres.py +++ b/auto_ontology/vdb/postgres.py @@ -215,7 +215,21 @@ def _get_engine(self) -> PGEngine: return self._engine def _table_exists(self) -> bool: - with psycopg.connect(self.connection_string) as conn: + # Bounded at every stage, since the chat worker reaches this from its + # warm-up before it reads questions: the connect (a blackholed database + # drops packets rather than refusing them), the query (a server that + # stops answering after connect), and a peer that silently disappears + # (TCP keepalives notice in about five seconds). A one-row catalog + # lookup has no business taking longer than three seconds anywhere. + with psycopg.connect( + self.connection_string, + connect_timeout=3, + keepalives=1, + keepalives_idle=3, + keepalives_interval=1, + keepalives_count=2, + options="-c statement_timeout=3000", + ) as conn: with conn.cursor() as cur: cur.execute( """ @@ -245,17 +259,20 @@ def _ensure_schema(self) -> None: ) conn.commit() - def _get_store(self) -> Optional[PGVectorStore]: + def _get_store(self, *, create_missing: bool = True) -> Optional[PGVectorStore]: """Return the vector store, creating the table on first write. - If the table doesn't exist and ``vector_size`` is not provided (e.g. - on read paths), returns ``None`` so callers can short-circuit. + With ``create_missing=False`` a collection that does not exist is left + alone and ``None`` is returned, for callers that must never write. """ if self._store is not None: return self._store + exists = self._table_exists() + if not exists and not create_missing: + return None engine = self._get_engine() - if not self._table_exists(): + if not exists: self._ensure_schema() engine.init_vectorstore_table( table_name=self.collection_name, @@ -339,6 +356,20 @@ def create_index(self, **kwargs: Any) -> str: self._get_store() return self.collection_name + def warm(self) -> bool: + """Open the connection pool and bind the store ahead of the first search. + + Otherwise the first search a process runs pays for the engine, a pooled + connection and the store's table inspection before it can query. + + Read-only, unlike :meth:`create_index`: a collection that does not + exist yet is left that way, and the first write creates it as before. + Existence is checked once, by the bind itself, so a collection dropped + a moment earlier cannot slip past a separate check and be recreated; + at worst binding fails and raises. Returns whether the store is ready. + """ + return self._get_store(create_missing=False) is not None + # ------------------------------------------------------------------ # Read / write # ------------------------------------------------------------------ diff --git a/auto_ontology/vdb/tests/test_records.py b/auto_ontology/vdb/tests/test_records.py index 310ee43c0..2338176b0 100644 --- a/auto_ontology/vdb/tests/test_records.py +++ b/auto_ontology/vdb/tests/test_records.py @@ -171,3 +171,30 @@ def _embed(texts: list[str], params: Any, input_type: str, **kw: Any) -> list: assert ( retriever.query("who buys", vdb_kwargs={"where": {"database_name": "x"}}) == [] ) + + +def test_warming_never_creates_a_collection(vdb: Any) -> None: + """Warming runs at worker start, before anything may have been ingested.""" + assert vdb.warm() is False + assert vdb._table_exists() is False + + write_embedded_rows(vdb, [_row("orders", _unit(0))]) + vdb.close() + + assert vdb.warm() is True + assert vdb.warm() is True + [hits] = vdb.retrieval([_unit(0)], top_k=1) + assert hits[0]["metadata"]["id"] == "orders" + + +def test_warming_a_collection_dropped_after_its_check_does_not_recreate_it( + vdb: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """The existence check says yes, then the table is gone by the time of binding.""" + monkeypatch.setattr(vdb, "_table_exists", lambda: True) + + with pytest.raises(Exception): + vdb.warm() + + monkeypatch.undo() + assert vdb._table_exists() is False