Skip to content
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,15 @@ AUTH_SECRET=myverylongsecret123
# Max concurrent LLM requests across all worker threads, to stay under the
# hosted endpoint's per-worker request cap (exceeding it surfaces as HTTP 503).
# LLM_MAX_INFLIGHT=6
# How long, in seconds, a connection to the LLM endpoint may sit unused before
# it is replaced rather than reused for the next call. Applies to models served
# through ChatNVIDIA. Lower it if the endpoint closes idle connections sooner.
# LLM_KEEPALIVE_IDLE_S=5

# --- Retrieval tuning (auto_ontology/utils/retriever.py) ---
# How many distinct query strings keep their embedding in memory, so a string
# searched for more than once is embedded once. About 16 KB each; 0 disables.
# AUTO_ONTOLOGY_QUERY_EMBED_CACHE_SIZE=1024

# --- Entity extraction tuning ---
# Max entities the LLM is asked to extract per question, for SQL generation
Expand Down
5 changes: 5 additions & 0 deletions CODE_OF_CONDUCT.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
<!--
SPDX-FileCopyrightText: Contributor Covenant, https://www.contributor-covenant.org
SPDX-License-Identifier: CC-BY-4.0
-->

# Code of Conduct

## Our Pledge
Expand Down
59 changes: 54 additions & 5 deletions auto_ontology/catalog/model/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,15 @@ def __init__(
self.columns_df.set_index("id")["full_name"].to_dict()
)

# This is for improving is_column_in_table
# This is for improving is_column_in_table. Read column-wise: one
# `iterrows()` pass builds a Series per row, which at catalog
# scale was half the cost of constructing a Schema.
self.columns_map = {}
for _, row in self.columns_df.iterrows():
if row["table_name_lower"] not in self.columns_map:
self.columns_map[row["table_name_lower"]] = {}
self.columns_map[row["table_name_lower"]][row["column_name_lower"]] = (
for table_name_lower, column_name_lower in zip(
self.columns_df["table_name_lower"],
self.columns_df["column_name_lower"],
):
self.columns_map.setdefault(table_name_lower, {})[column_name_lower] = (
True
)

Expand Down Expand Up @@ -151,6 +154,52 @@ def reset_columns_props(self):
axis=1,
)

def column_names_by_table(self) -> dict[str, tuple[str, ...]]:
"""Lower-cased column names per lower-cased table name, in row order.

This is what SQL extraction needs from the catalog on every parse: the
column set of each table, to hand to sqlglot's ``qualify`` and to
attribute bare column references. Reading it off ``columns_df`` row by
row costs a pandas Series per column of the *whole* schema, per parse,
which made a parse scale with the catalog instead of with the query.
It is derived once and kept for as long as ``columns_df`` is the same
frame with the same number of rows.

Names are lower-cased with ``str(...).lower()`` and nothing else, on
purpose: unlike ``columns_map`` they keep any surrounding quotes, which
is the spelling the extractor has always matched against.
"""
df = self.columns_df
if df is None or df.empty:
return {}
# The frame itself is held, not its id(): an id can be handed to a new
# object once the old one is collected, and would then vouch for an
# index read from a frame that no longer exists.
cached = getattr(self, "_column_names_by_table", None)
if cached is not None and cached[0] is df and cached[1] == len(df):
return cached[2]
grouped: dict[str, list[str]] = {}
for table_name, column_name in zip(df["table_name"], df["column_name"]):
grouped.setdefault(str(table_name).lower(), []).append(
str(column_name).lower()
)
index = {table: tuple(columns) for table, columns in grouped.items()}
self._column_names_by_table = (df, len(df), index)
return index

def forget_nodes(self) -> None:
"""Drop the table and column nodes built by earlier lookups.

``get_table_node`` and ``get_column_node`` build a node on first use and
keep it. A schema that is parsed against repeatedly calls this between
parses so that each one resolves names as the first did; the frames,
``columns_map`` and ``id_to_node`` it resolves them from are untouched.
"""
self.table_nodes = {}
self.column_nodes = {}
self.tables_to_columns = {}
self.tables_columns_pos = {}

def get_schema_name(self):
return self.schema_name

Expand Down
129 changes: 113 additions & 16 deletions auto_ontology/catalog/parsers/sqlglot_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,13 @@
from __future__ import annotations

from dataclasses import dataclass, field
from typing import Any

import sqlglot
from sqlglot import exp
from sqlglot.errors import ParseError, TokenError
from sqlglot.optimizer.qualify import qualify
from sqlglot.schema import MappingSchema


class SQLSyntaxError(ValueError):
Expand Down Expand Up @@ -151,6 +153,54 @@ def _alias_to_table_map(
return mapping


class SchemaCatalog(dict):
"""A ``{schema_name: Schema}`` map that is parsed against more than once.

Extraction derives two things from the catalog that depend on the catalog
alone, not on the SQL: the flat ``{table: {column: type}}`` mapping
``qualify`` reads, and the ``MappingSchema`` sqlglot normalises that
mapping into for one dialect. Both are linear in the number of columns in
the *whole* catalog, so rebuilding them per statement makes every parse
pay for every table nobody asked about.

A plain ``dict`` has nowhere to keep them. Wrapping the map in this class
is how a caller says the same catalog will be reused, and that it will not
add, drop or swap a schema in place afterwards -- build a new
``SchemaCatalog`` for a changed catalog instead. A plain ``dict`` still
works everywhere and is simply derived from each time.

Not safe for concurrent parses on its own: these caches, like the lazily
built nodes on each ``Schema``, are filled on first use. Callers sharing
one across threads serialise the parse.
"""

__slots__ = ("_qualify_columns", "_qualify_schemas")

def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self._qualify_columns: dict[str, dict[str, str]] | None = None
self._qualify_schemas: dict[str, MappingSchema] = {}


def _schema_column_index(schema: Any) -> dict[str, tuple[str, ...]]:
"""``{table: (column, ...)}`` for one catalog schema, names lower-cased.

A :class:`~auto_ontology.catalog.model.Schema` keeps this itself; anything
else that merely looks like one (a ``columns_df`` and nothing more) has it
derived here, the same way.
"""
accessor = getattr(schema, "column_names_by_table", None)
if callable(accessor):
return accessor()
df = getattr(schema, "columns_df", None)
if df is None or df.empty:
return {}
grouped: dict[str, list[str]] = {}
for table_name, column_name in zip(df["table_name"], df["column_name"]):
grouped.setdefault(str(table_name).lower(), []).append(str(column_name).lower())
return {table: tuple(columns) for table, columns in grouped.items()}


def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]:
"""Build a flat ``{table: {col: "TEXT"}}`` dict for sqlglot's qualify pass.

Expand All @@ -161,18 +211,64 @@ def _build_schema_dict(all_schemas: dict) -> dict[str, dict[str, str]]:
Multi-schema disambiguation is handled separately via ``alias_map`` and
``source_table_names`` (which use fully-qualified keys).
"""
if isinstance(all_schemas, SchemaCatalog) and (
all_schemas._qualify_columns is not None
):
return all_schemas._qualify_columns

schema: dict[str, dict[str, str]] = {}
for s in all_schemas.values():
df = s.columns_df
if df is None or df.empty:
continue
for _, row in df.iterrows():
tbl = str(row["table_name"]).lower()
col = str(row["column_name"]).lower()
schema.setdefault(tbl, {})[col] = "TEXT"
for tbl, columns in _schema_column_index(s).items():
table_columns = schema.setdefault(tbl, {})
for col in columns:
table_columns[col] = "TEXT"

if isinstance(all_schemas, SchemaCatalog):
all_schemas._qualify_columns = schema
return schema


# Memo tables sqlglot's MappingSchema fills during a parse, keyed by the query's
# own table expressions (aliases included) and names. They are lookups only:
# clearing one makes the next lookup recompute what it would have returned.
_MAPPING_SCHEMA_QUERY_CACHES = (
"_normalized_table_cache",
"_find_cache",
"_normalized_name_cache",
)


def _qualify_schema(
all_schemas: dict, schema_dict: dict[str, dict[str, str]], dialect: str
) -> "MappingSchema | dict[str, dict[str, str]]":
"""What to hand ``qualify`` as its schema for *dialect*.

``qualify`` turns a mapping into ``MappingSchema(mapping, dialect=dialect)``
before it does anything else, and passes an existing schema object through
untouched. For a :class:`SchemaCatalog` that object is built once per
dialect here, by that exact call, and reused; sqlglot only reads from it.
Everything else gets the mapping, as before.

A reused one has its per-query memo tables emptied first. sqlglot keys them
by the parsed query's table expressions, so a schema kept for the life of a
worker would otherwise hold an entry for every alias any query ever used:
2,000 parses left 4,000 entries in each. What is worth keeping -- the
catalog mapping, normalised once at construction -- is not among them.
"""
if not isinstance(all_schemas, SchemaCatalog) or not schema_dict:
return schema_dict
cached = all_schemas._qualify_schemas.get(dialect)
if cached is None:
cached = MappingSchema(schema_dict, dialect=dialect)
all_schemas._qualify_schemas[dialect] = cached
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
else:
for name in _MAPPING_SCHEMA_QUERY_CACHES:
memo = getattr(cached, name, None)
if isinstance(memo, dict):
memo.clear()
return cached


_BINARY_CMP = (exp.EQ, exp.GT, exp.GTE, exp.LT, exp.LTE, exp.NEQ)

_OP_SYMBOL: dict[type, str] = {
Expand Down Expand Up @@ -692,7 +788,7 @@ def extract_tables_and_columns(
qualified = qualify(
statement,
dialect=dialect,
schema=schema_dict,
schema=_qualify_schema(all_schemas, schema_dict, dialect),
qualify_columns=True,
validate_qualify_columns=False,
expand_stars=bool(schema_dict),
Expand All @@ -718,13 +814,11 @@ def extract_tables_and_columns(
if unresolved and all_schemas:
col_to_tables: dict[str, list[str]] = {}
for schema_name, s in all_schemas.items():
df = s.columns_df
if df is None or df.empty:
continue
skey = schema_name.lower()
for _, row in df.iterrows():
col_n = str(row["column_name"]).lower()
tbl_n = str(row["table_name"]).lower()
# Per table, not per column: which source table a catalog table
# stands for is decided by its name alone, so only the columns of
# tables this query actually reads are ever walked.
for tbl_n, columns in _schema_column_index(s).items():
# Try the qualified name first (schema.table); fall back to bare
# table name for SQL that doesn't prefix tables with a schema.
qualified_name = f"{skey}.{tbl_n}"
Expand All @@ -733,8 +827,11 @@ def extract_tables_and_columns(
if qualified_name in source_table_names
else (tbl_n if tbl_n in source_table_names else None)
)
if matched and matched not in col_to_tables.get(col_n, []):
col_to_tables.setdefault(col_n, []).append(matched)
if not matched:
continue
for col_n in columns:
if matched not in col_to_tables.get(col_n, []):
col_to_tables.setdefault(col_n, []).append(matched)

for col_name in unresolved:
candidates = col_to_tables.get(col_name, [])
Expand Down
89 changes: 89 additions & 0 deletions auto_ontology/dal/datasources.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@
import pandas as pd
from sqlalchemy import (
ColumnElement,
Numeric,
Text,
and_,
case,
cast,
column,
distinct,
func,
Expand Down Expand Up @@ -251,6 +253,93 @@ def fetch_schemas_by_ids(
return [dict(r) for r in store().query_read(statement)]


def _schema_row_hash() -> ColumnElement:
"""One 64-bit hash per row of the ``fetch_schemas_by_ids`` result.

Over exactly the seven fields that query selects, and nothing else: a
description edit or a new sample value leaves it alone, while a renamed
column, a retyped one, or a table added, dropped or moved does not.

``data_type`` goes last because it is the only nullable field and
``concat_ws`` skips nulls: at the end, a missing type shortens the string
rather than shifting another field into its place.
"""
return func.hashtextextended(
func.concat_ws(
# The unit separator, which no identifier contains, so two rows
# cannot hash alike by splitting the same text at different points.
literal("\x1f"),
s.catalog_column.c.id,
s.catalog_column.c.name,
s.catalog_table.c.id,
s.catalog_table.c.name,
s.catalog_schema.c.name,
s.catalog_database.c.name,
s.catalog_column.c.data_type,
),
0,
)


def _schema_fingerprint(row_count: int, digest: Any) -> str:
"""How a row count and a sum of row hashes are written down.

One place, because the two functions below must agree to the character:
one gets the sum from Postgres as a ``Decimal``, the other adds the hashes
up itself.
"""
return f"{int(row_count)}:{int(digest)}"


def fetch_catalog_schema_fingerprint() -> str:
"""A digest that changes whenever ``fetch_schemas_by_ids()`` would.

Lets a caller keep what it built from the whole-catalog rows and learn,
for the price of one aggregate, whether it is still current. The rows are
hashed where they live, so nothing but the answer crosses the wire.

The sum makes it independent of row order, and the count keeps an added
row from being cancelled by a removed one whose hashes happen to offset.
Compare it only with the fingerprint
:func:`fetch_catalog_schema_snapshot` returned alongside the rows.
"""
statement = select(
func.count().label("row_count"),
# Numeric: summing bigint hashes overflows bigint long before the
# catalog is large.
func.coalesce(func.sum(cast(_schema_row_hash(), Numeric)), 0).label("digest"),
).select_from(_catalog_join())

row = store().query_read(statement)[0]
return _schema_fingerprint(row["row_count"], row["digest"])


def fetch_catalog_schema_snapshot() -> tuple[str, list[dict[str, str]]]:
"""Every row ``fetch_schemas_by_ids()`` returns, with their fingerprint.

The fingerprint is computed from the rows returned, which one statement
read from one snapshot of the database. Reading the fingerprint and the
rows separately cannot promise that: a commit landing between the two
pairs a fingerprint with rows it does not describe, and because the
fingerprint is a function of content, a catalog that later returns to the
earlier state would match it again and vouch for the wrong rows.
"""
statement = select(
s.catalog_column.c.name.label("column_name"),
s.catalog_column.c.id.label("column_id"),
s.catalog_table.c.name.label("table_name"),
s.catalog_table.c.id.label("table_id"),
s.catalog_database.c.name.label("database_name"),
s.catalog_schema.c.name.label("table_schema"),
s.catalog_column.c.data_type,
_schema_row_hash().label("row_hash"),
).select_from(_catalog_join())

rows = [dict(r) for r in store().query_read(statement)]
digest = sum(row.pop("row_hash") for row in rows)
return _schema_fingerprint(len(rows), digest), rows


# ---------------------------------------------------------------------------
# Table
# ---------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions auto_ontology/dal/tests/dal_surface.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@
"fetch_all_schema_ids": "() -> 'list[str]'",
"fetch_all_tables_without_term": "(database_name: 'str | None' = None) -> 'list[dict[str, Any]]'",
"fetch_bridge_table_candidates": "(database_name: 'str') -> 'list[dict[str, Any]]'",
"fetch_catalog_schema_fingerprint": "() -> 'str'",
"fetch_catalog_schema_snapshot": "() -> 'tuple[str, list[dict[str, str]]]'",
"fetch_col_table_contexts": "(col_ids: 'list[str]') -> 'dict[str, dict[str, str]]'",
"fetch_columns_for_table": "(table_id: 'str', *, skip: 'int' = 0, limit: 'int | None' = None) -> 'dict[str, Any] | None'",
"fetch_databases": "(zone_ids: 'list[str] | None' = None) -> 'list[dict[str, Any]]'",
Expand Down
Loading
Loading