Skip to content
Merged
Show file tree
Hide file tree
Changes from 33 commits
Commits
Show all changes
80 commits
Select commit Hold shift + click to select a range
4131598
Add a ReAct text-to-SQL agent that searches the ontology with tools
ftatiana-nv Nov 3, 2026
3ad349d
Name near-copy tables in the ReAct retrieve output
ftatiana-nv Nov 3, 2026
a3e59aa
Give ReAct tables their Term descriptions
ftatiana-nv Nov 3, 2026
b5e4f63
Enable the ReAct text-to-SQL agent by default
liavnave Nov 4, 2026
643ce96
Default the non-reasoning model to Claude Sonnet
liavnave Nov 4, 2026
a9c4ee9
Raise the ReAct agent's default episode limits
liavnave Nov 4, 2026
2691fde
Keep semantic FKs inside their table family
ftatiana-nv Nov 4, 2026
608afcd
Ask the ReAct agent for exactly the requested output fields
ftatiana-nv Nov 4, 2026
7d69950
Search tables and columns with their terms and attributes
ftatiana-nv Nov 4, 2026
ec073bf
Default the nvapi- embedding model to nemotron-3-embed-1b
tomer-levin-nv Nov 4, 2026
8ccd111
Guide SQL generation to build multi-part questions with CTEs and keep…
tomer-levin-nv Nov 4, 2026
31f2681
Run text-to-SQL through the ReAct agent only, without LangGraph
ftatiana-nv Nov 5, 2026
8f65cf6
Name the text-to-SQL and entity-coverage pipeline parts steps, not nodes
ftatiana-nv Nov 5, 2026
c57f029
Answer forecasting questions with a KumoRFM predict tool in the ReAct…
liavnave Nov 10, 2026
85094da
Let the ReAct agent grep a run_sql result too long to show, behind TE…
ftatiana-nv Nov 10, 2026
0b5afbf
Let the ReAct agent answer with run_sql(final=true) instead of submit…
ftatiana-nv Nov 10, 2026
a197c17
Check ReAct SQL against the catalog of the database it runs on
liavnave Nov 11, 2026
37ee83a
Reject ReAct SQL on tables no retrieve result named
liavnave Nov 11, 2026
a0dabaa
Bound the ReAct sibling row counts in time and number
liavnave Nov 11, 2026
8245f58
Count repeated ReAct tool calls against the budget
liavnave Nov 11, 2026
f859e39
Search the semantic layer in the ReAct retrieve tool
ftatiana-nv Nov 11, 2026
bb03fe3
Pass a debug turn's rejected SQL and feedback to the SQL generator
liavnave Nov 11, 2026
8b8aace
Keep other databases' tables out of a retrieve's table limit
liavnave Nov 11, 2026
aa4448c
Retry every vector search on a transient embedding error
ftatiana-nv Nov 11, 2026
678bcf1
Give the ReAct agent the organization's custom prompts and acronyms
liavnave Nov 11, 2026
d55b7db
Describe every table and column at the end of a compilation
ftatiana-nv Nov 11, 2026
e11ea37
Show only names and descriptions on retrieve's tables and columns
ftatiana-nv Nov 11, 2026
a9073cc
Copy semantic descriptions onto tables and columns that have none
ftatiana-nv Nov 12, 2026
5455891
Re-embed attributes and Terms when their column or table description …
ftatiana-nv Nov 12, 2026
65b4fec
Shorten retrieve tables that don't fit and name what was left out
ftatiana-nv Nov 12, 2026
f423b56
Answer information questions in words through submit
liavnave Nov 15, 2026
bda496e
Look information questions up as themselves, and answer them only fro…
ftatiana-nv Nov 15, 2026
3fc4e08
Shorten the information opening and name the tool variants by what th…
ftatiana-nv Nov 15, 2026
522f779
Offer grep by default
ftatiana-nv Nov 15, 2026
190b6aa
Let submit leave sql out
ftatiana-nv Nov 15, 2026
1381fed
Accept an answer without SQL only after an information lookup
ftatiana-nv Nov 15, 2026
acb8918
Keep a crossing semantic FK when its twin cannot be checked
ftatiana-nv Nov 15, 2026
01ba4e9
Type the test helpers in the information and table-family tests
ftatiana-nv Nov 15, 2026
397324c
Show columns without an attribute in the information view
ftatiana-nv Nov 15, 2026
7f2b617
Skip attributes of another database in the information view
ftatiana-nv Nov 16, 2026
008dab6
Keep an oversized first block in the information view, cut to fit
ftatiana-nv Nov 16, 2026
097d4c9
Do not list an already shown table as also found in retrieve
ftatiana-nv Nov 16, 2026
9e255b6
security, sql should be read-only
DinaLaptii Nov 16, 2026
59b14e8
Type the retrieve test fixture helpers
ftatiana-nv Nov 16, 2026
12a25ec
Keep retrieve's omission notice inside the observation limit
ftatiana-nv Nov 16, 2026
ad57f89
Bound grep's regex matching with a time budget
ftatiana-nv Nov 16, 2026
279db42
Turn the grep tool off by default
ftatiana-nv Nov 16, 2026
a3d4795
Look up the twin column by table id, not by name across schemas
DinaLaptii Nov 16, 2026
10d4223
live.NEW_CUSTOMERS and archive.NEW_CUSTOMERS no longer overwrite each…
DinaLaptii Nov 16, 2026
6881e7a
Mark the information lookup done only after it returns
ftatiana-nv Nov 16, 2026
a43be0b
Merge remote-tracking branch 'origin/main' into feat/react-ontology-a…
liavnave Nov 16, 2026
9538208
Merge remote-tracking branch 'origin/feat/react-ontology-agent_clean'…
liavnave Nov 16, 2026
bedcbe2
Match grep against whole cells, not their shortened display
ftatiana-nv Nov 16, 2026
1f058fc
Type the pagila fixture in the read-only Postgres tests
ftatiana-nv Nov 16, 2026
9c80cdf
added timeout
DinaLaptii Nov 16, 2026
caad32e
Revert semantic folder to main
liavnave Nov 16, 2026
1c541a6
Stop grep once its shared time budget is spent
ftatiana-nv Nov 16, 2026
c30c1fb
revert
DinaLaptii Nov 16, 2026
61a4c46
fix
DinaLaptii Nov 16, 2026
3780e10
Allow one answer attempt per reply after the tool-call budget
liavnave Nov 16, 2026
ed7f227
Remove unused fetch_column_attributes_for_embedding
liavnave Nov 16, 2026
f69caec
Annotate the attribute embedding select helpers
liavnave Nov 16, 2026
a6dbbcf
Annotate the sql_fragments reference helpers
liavnave Nov 16, 2026
21a03f9
Keep a database with no schemas from borrowing every other database's
ftatiana-nv Nov 16, 2026
8730ebd
Remove unused delete_semantic_fk and fill_empty_descriptions
tomer-levin-nv Nov 16, 2026
822b3d1
Re-embed a Term with the embed_term signature it has
tomer-levin-nv Nov 16, 2026
8a5de55
Match ReAct SQL tables to the schema retrieve showed, not only the name
DinaLaptii Nov 16, 2026
7097c1f
Annotate the debug-evidence grounding tests
liavnave Nov 16, 2026
0c8b1ec
Refuse tables in a catalog other than the connector's
ftatiana-nv Nov 16, 2026
8ac8145
Revert semantic-layer write and embedding changes to main
tomer-levin-nv Nov 16, 2026
04186e8
Tag the columns retrieve matched
ftatiana-nv Nov 16, 2026
dac4d1e
Count only the SQL a definition shows as in view
ftatiana-nv Nov 16, 2026
8716f10
Search a one-word query by physical name
ftatiana-nv Nov 16, 2026
211e9ce
Show the submitted SQL in the previous-attempt note
DinaLaptii Nov 16, 2026
2b4e533
Classify embedding retries by error type, not message text
ftatiana-nv Nov 16, 2026
c325a8e
Pass organization instructions and glossary to PQL generation
liavnave Nov 16, 2026
95d0bee
Match table names in view at SQL identifier boundaries
DinaLaptii Nov 16, 2026
f821113
fix
DinaLaptii Nov 16, 2026
0509ec0
Count only semantic Terms when deciding a table is covered
tomer-levin-nv Nov 16, 2026
f6fac80
Let fetch_table_column_names raise instead of returning an empty catalog
tomer-levin-nv Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion THIRD_PARTY_NOTICES.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ build artifact — see `pyproject.toml`, `frontend/package.json`, and the
| langchain-nvidia-ai-endpoints | >=1.4.3 | MIT | https://github.com/langchain-ai/langchain-nvidia |
| langchain-openai | >=1.3.0,<1.3.5 | MIT | https://github.com/langchain-ai/langchain |
| langchain-postgres | >=0.0.17 | MIT | https://github.com/langchain-ai/langchain-postgres |
| LangGraph | >=1.2.11 | MIT | https://github.com/langchain-ai/langgraph |
| mysql-connector-python | >=26.7.0 | GPL-2.0 with Universal FOSS Exception — see note below | https://github.com/mysql/mysql-connector-python |
| NumPy | >=2.5.2 | BSD-3-Clause AND 0BSD AND MIT AND Zlib AND CC0-1.0 | https://github.com/numpy/numpy |
| openai | ==2.44.0 | Apache-2.0 | https://github.com/openai/openai-python |
Expand Down
2 changes: 1 addition & 1 deletion auto_ontology/connectors/clickhouse.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
:attr:`~ClickHouseDatabase.database_name` and the ``table_schema`` every
introspection method reports, and only its tables are ingested. Reporting
another database's tables under this connection would make
``auto_ontology.retrieval.text_to_sql.formatters_util.qualify_table`` emit a
schema-qualified SQL emit a
three-part ``bound.other.table``, which ClickHouse cannot parse -- so a
second database is a second connection.
* **``system`` tables, not ``information_schema``.** ClickHouse ships an
Expand Down
235 changes: 204 additions & 31 deletions auto_ontology/dal/datasources.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,18 @@
distinct,
func,
literal,
literal_column,
select,
update,
)
from sqlalchemy import values as sa_values
from sqlalchemy.dialects.postgresql import aggregate_order_by

from auto_ontology.dal import schema as s
from auto_ontology.dal.session import store, write_transaction
from auto_ontology.dal.sql_fragments import (
column_description_expr,
column_reference_expr,
table_description_expr,
)
from auto_ontology.dal.tags import TARGET_COLUMN, TARGET_TABLE, fetch_tags_map
Expand Down Expand Up @@ -954,36 +957,17 @@ def fetch_columns_for_table(
return table


def fetch_tables_by_ids(table_ids: list[str]) -> list[dict[str, Any]]:
"""Tables with a name/type/description/sample_values summary of each column.

Returns ``[]`` rather than raising if the query fails: this decorates
retrieval results, and losing the decoration beats losing the results.

A table with no columns does not appear at all — the column join is inner.
Left that way deliberately: a column-less table in the catalog is a symptom
worth seeing where it originates, not something to paper over here.
def _semantic_fk_target_expr() -> ColumnElement:
"""``schema.table.column`` a catalog column's semantic FK lands on.

``sample_values`` is what lets a SQL-generating model write a correct
``->``/``->>`` path into a JSONB column instead of guessing a plausible
sibling key — this is the only per-column field ``candidates_preparation``'s
"back-fill" step (§4a: any table whose columns arrived without
sample_values from the vector-index hit) exists to supply. It was
selected here from Aug 2026 until the Neo4j-to-Postgres port silently
dropped it from this query's rewrite (the Neo4j Cypher version had it);
restored so the back-fill isn't a no-op again. ``is_nullable`` is included
alongside it so projection rules can tell when a requested item may be
missing; ``None`` means the connector could not determine it.
``semantic_fk`` resolves the referenced attribute back to its owning column
and returns that target as ``schema.table.column`` for SQL prompts.
The FK points at an attribute; the column that owns the attribute is the
target. Correlated on ``catalog_column``.
"""
if not table_ids:
return []
semantic_owner = s.column__has_attribute.alias("semantic_fk_owner")
semantic_target = s.catalog_column.alias("semantic_fk_target")
semantic_table = s.catalog_table.alias("semantic_fk_table")
semantic_schema = s.catalog_schema.alias("semantic_fk_schema")
semantic_fk_target = (
return (
select(
func.concat_ws(
".",
Expand Down Expand Up @@ -1020,12 +1004,38 @@ def fetch_tables_by_ids(table_ids: list[str]) -> list[dict[str, Any]]:
.limit(1)
.scalar_subquery()
)


def fetch_tables_by_ids(table_ids: list[str]) -> list[dict[str, Any]]:
"""Tables with a name/type/description/sample_values summary of each column.

Returns ``[]`` rather than raising if the query fails: this decorates
retrieval results, and losing the decoration beats losing the results.

A table with no columns does not appear at all — the column join is inner.
Left that way deliberately: a column-less table in the catalog is a symptom
worth seeing where it originates, not something to paper over here.

``sample_values`` is what lets a SQL-generating model write a correct
``->``/``->>`` path into a JSONB column instead of guessing a plausible
sibling key. It was selected here from Aug 2026 until the
Neo4j-to-Postgres port silently dropped it from this query's rewrite (the
Neo4j Cypher version had it); restored so a table whose columns arrived
without sample_values from the vector-index hit can be back-filled. ``is_nullable`` is included
alongside it so projection rules can tell when a requested item may be
missing; ``None`` means the connector could not determine it.
``semantic_fk`` resolves the referenced attribute back to its owning column
and returns that target as ``schema.table.column`` for SQL prompts.
"""
if not table_ids:
return []
semantic_fk_target = _semantic_fk_target_expr()
try:
rows = store().query_read(
select(
s.catalog_table.c.id,
s.catalog_table.c.name,
s.catalog_table.c.description,
table_description_expr().label("description"),
s.catalog_table.c.pk,
s.catalog_database.c.name.label("database_name"),
s.catalog_schema.c.name.label("schema_name"),
Expand Down Expand Up @@ -1086,6 +1096,50 @@ def fetch_tables_by_ids(table_ids: list[str]) -> list[dict[str, Any]]:
return list(tables.values())


def fetch_table_column_names(database_name: str | None) -> list[dict[str, Any]]:
"""Every table in *database_name* (all databases when ``None``) with its
column names: ``{id, name, schema_name, database_name, columns}``.

Names only, so a whole catalog is cheap to compare table against table.
Returns ``[]`` if the query fails, like :func:`fetch_tables_by_ids`.
"""
query = select(
s.catalog_table.c.id,
s.catalog_table.c.name,
s.catalog_database.c.name.label("database_name"),
s.catalog_schema.c.name.label("schema_name"),
s.catalog_column.c.name.label("column_name"),
).select_from(
_table_join().join(
s.catalog_column, s.catalog_column.c.table_id == s.catalog_table.c.id
)
)
if database_name:
query = query.where(s.catalog_database.c.name == database_name)
try:
rows = store().query_read(
query.order_by(s.catalog_table.c.id, s.catalog_column.c.ordinal_position)
)
except Exception:
logger.warning("fetch_table_column_names: query failed", exc_info=True)
return []
Comment thread
tomer-levin-nv marked this conversation as resolved.
Resolved

tables: dict[str, dict[str, Any]] = {}
for row in rows:
table = tables.setdefault(
str(row["id"]),
{
"id": str(row["id"]),
"name": row["name"] or "",
"schema_name": row["schema_name"] or "",
"database_name": row["database_name"] or "",
"columns": [],
},
)
table["columns"].append(row["column_name"])
return list(tables.values())


def fetch_table_context(table_id: str) -> dict[str, Any]:
"""``{columns, fks}`` for one table — what the SQL generator is handed.

Expand Down Expand Up @@ -1180,6 +1234,7 @@ def fetch_tables_and_columns_by_node_ids(
s.catalog_column.c.name.label("column_name"),
s.catalog_column.c.data_type,
column_description_expr().label("description"),
column_reference_expr().label("references"),
s.catalog_column.c.sample_values,
s.catalog_database.c.name.label("database_name"),
)
Expand All @@ -1198,12 +1253,7 @@ def fetch_tables_and_columns_by_node_ids(
s.catalog_table.c.name.label("table_name"),
s.catalog_schema.c.name.label("table_schema"),
s.catalog_table.c.table_type,
# The table's own description, not the fallback -- unlike
# the columns three lines earlier. Asymmetric on purpose:
# these frames feed embeddings, and widening what gets
# embedded would shift retrieval results with no test
# noticing.
s.catalog_table.c.description,
table_description_expr().label("description"),
s.catalog_database.c.name.label("database_name"),
)
.select_from(_table_join())
Expand All @@ -1220,6 +1270,129 @@ def fetch_tables_and_columns_by_node_ids(
return tables_df, columns_df, database_name


def fetch_catalog_node_ids(database_name: str) -> list[str]:
"""Every table and column id in *database_name*, tables first."""
tables = store().query_read(
select(s.catalog_table.c.id)
.select_from(_table_join())
.where(s.catalog_database.c.name == database_name)
.order_by(s.catalog_table.c.id)
)
columns = store().query_read(
select(s.catalog_column.c.id)
.select_from(_catalog_join())
.where(s.catalog_database.c.name == database_name)
.order_by(s.catalog_column.c.id)
)
return [str(r["id"]) for r in tables] + [str(r["id"]) for r in columns]


def fetch_description_context(database_name: str) -> list[dict[str, Any]]:
"""Every table in *database_name* with its columns, as descriptions resolve.

``description`` is what a reader sees: the node's own, else the Term or
attribute behind it (see :mod:`auto_ontology.dal.sql_fragments`), and
``own_description`` the node's own alone. A table also carries the
descriptions of the bridge SqlAttributes written over it, and a column the
``schema.table.column`` its foreign key lands on.
"""
bridge_descriptions = (
select(
func.string_agg(
s.sql_attribute.c.description,
aggregate_order_by(literal_column("' '"), s.sql_attribute.c.name),
)
)
.select_from(
s.sql_attribute.join(
s.sql_attribute__sql,
s.sql_attribute__sql.c.attribute_id == s.sql_attribute.c.id,
).join(
s.sql_query__table,
s.sql_query__table.c.sql_query_id
== s.sql_attribute__sql.c.sql_query_id,
)
)
.where(
s.sql_attribute.c.source == SQL_ATTR_SOURCE_BRIDGE,
s.sql_query__table.c.table_id == s.catalog_table.c.id,
)
.scalar_subquery()
)
tables = store().query_read(
select(
s.catalog_table.c.id,
s.catalog_table.c.name,
s.catalog_schema.c.name.label("schema_name"),
table_description_expr().label("description"),
s.catalog_table.c.description.label("own_description"),
bridge_descriptions.label("bridge_description"),
)
.select_from(_table_join())
.where(s.catalog_database.c.name == database_name)
.order_by(s.catalog_schema.c.name, s.catalog_table.c.name)
)
columns = store().query_read(
select(
s.catalog_column.c.id,
s.catalog_column.c.table_id,
s.catalog_column.c.name,
s.catalog_column.c.data_type,
s.catalog_column.c.sample_values,
column_description_expr().label("description"),
s.catalog_column.c.description.label("own_description"),
column_reference_expr().label("references"),
)
.select_from(_catalog_join())
.where(s.catalog_database.c.name == database_name)
.order_by(s.catalog_column.c.table_id, s.catalog_column.c.id)
)
by_table: dict[str, list[dict[str, Any]]] = {}
for row in columns:
by_table.setdefault(str(row["table_id"]), []).append(
{
**dict(row),
"sample_values": stringify_sample_values(row["sample_values"]) or [],
}
)
return [
{**dict(table), "columns": by_table.get(str(table["id"]), [])}
for table in tables
]


def fill_empty_descriptions(
table_descriptions: dict[str, str],
column_descriptions: dict[str, str],
) -> list[str]:
"""Write each description onto its table or column only where none is set.

A description that came with the source metadata, or that a user wrote, is
never replaced. Returns the ids actually written.
"""
written: list[str] = []
with write_transaction():
for table, descriptions in (
(s.catalog_table, table_descriptions),
(s.catalog_column, column_descriptions),
):
for node_id, description in descriptions.items():
text = (description or "").strip()
if not text:
continue
rows = store().query_write(
update(table)
.where(
table.c.id == node_id,
func.trim(func.coalesce(table.c.description, "")) == "",
)
.values(description=text)
.returning(table.c.id)
)
written.extend(str(row["id"]) for row in rows)
return written


def fetch_bridge_table_candidates(database_name: str) -> list[dict[str, Any]]:
"""Pure-FK junction tables eligible for a bridge SqlAttribute.

Expand Down
19 changes: 4 additions & 15 deletions auto_ontology/dal/model_interchange.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
build_custom_analysis_semantic_row,
build_sql_attribute_semantic_row,
build_table_data_row,
build_term_semantic_rows,
)
from auto_ontology.server.model_interchange.schemas import (
AutoOntologyModelDocument,
Expand Down Expand Up @@ -1385,20 +1384,8 @@ def _import_terms(
represents_rows.append({"term_id": live_term_id, "table_id": live_table_id})
_link(s.table__term, represents_rows)

if embed_buffer is not None and newly_created:
db_names = _database_names_for_terms(newly_created)
for term in terms:
live_term_id, was_created = term_results[term.id]
if not was_created:
continue
embed_buffer.semantic_rows.extend(
build_term_semantic_rows(
database_name=db_names.get(live_term_id, ""),
live_id=live_term_id,
name=term.name,
description=term.description,
),
)
if embed_buffer is not None:
embed_buffer.term_ids.extend(newly_created)
Comment thread
ftatiana-nv marked this conversation as resolved.
Resolved


def _import_column_attributes(
Expand Down Expand Up @@ -1481,6 +1468,8 @@ def _import_column_attributes(
is_unique=col_ctx.is_unique,
sample_values=col_ctx.sample_values,
schema_name=col_ctx.schema_name,
column_description=col_ctx.description,
column_data_type=col_ctx.data_type,
),
)

Expand Down
Loading