Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
f75ae87
test(contracts): the case matrix of the interface-contract foundations
Nov 15, 2026
06ed99a
feat(contracts): interface contracts, kits, manifests and their checks
Nov 15, 2026
0d54bfa
test(contracts): the case matrix of the envelope, N and N-1, idempote…
Nov 15, 2026
4fdb62a
feat(contracts): call envelope, N and N-1 kit runs, idempotency and t…
Nov 15, 2026
0a9183b
refactor(contracts): one place decides what an operation is and when …
Nov 15, 2026
219c8e8
ci(repo): the interface checks stay optional until main's protection …
Nov 15, 2026
7b46943
fix(contracts): close the review findings on refusals, compatibility,…
Nov 15, 2026
ffad664
fix(store): watch signals with a selector, so 1,024 open files never …
Nov 15, 2026
91bb4c2
feat(contracts): aware datetimes cross exactly, and envelopes come fr…
Nov 15, 2026
20f1330
build(repo): pin pydantic-core in the runtime group, now a declared d…
Nov 15, 2026
6ba558b
feat(contracts): a kit case may run in a process of its own, built fr…
Nov 15, 2026
7b871a8
refactor(testing): the shared test helpers become the test-only orvex…
Nov 15, 2026
b9ad795
test(store): the case matrix of the store's interface contract and it…
Nov 15, 2026
979e239
refactor(store): consumers reach the store only through its interface…
Nov 15, 2026
bb9a229
test(server): fault injections at the store binding reach no internals
Nov 15, 2026
d40bdb2
refactor(store): Arrow crosses the store contract as the foundations'…
Nov 15, 2026
396b2aa
build(store): the store's tests declare orvex-testing, and its contra…
Nov 15, 2026
1922be5
refactor(checks): tests, runners and tools reach the server only thro…
Nov 15, 2026
8e7f0ef
feat(store): each operation declares only the codes it raises, and th…
Nov 15, 2026
5fbab76
fix(store): the store's face is its api, test dependencies keep their…
Nov 15, 2026
3b7e927
feat(store): the extension and engine-version codes are declared, pro…
Nov 15, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci-alerts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ on:
- property-nightly
- ossie-upstream
- ci-alerts-selftest
- interfaces
types: [completed]
schedule:
- cron: "41 */6 * * *"
Expand Down
181 changes: 181 additions & 0 deletions .github/workflows/interfaces.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
# Interface contracts (#486): their record is current and compatible with the last release
# (G3), and a module installs and passes its own tests alone, outside the repository (G4).
name: interfaces

on:
pull_request:
push:
branches: [main]

# A newer commit on a pull request cancels the pull request's older runs (#386). Pushes to one
# branch (main) share a group: a running one is never cancelled, and only the newest waiting
# one is kept, since it tests everything the older ones would (#391).
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
contents: read

jobs:
interface-contracts:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# The whole history and its tags: the check reads the record of the last release.
fetch-depth: 0
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: "3.15"
- name: Install from the lock file and the contracts package
run: |
python -m pip install --quiet pip==26.3
python -m pip install --quiet -r pylock.toml
python -m pip install --quiet --no-deps --no-build-isolation -e packages/contracts
- name: The interface record is current
run: python -m tools.contracts.interface_record --check
- name: Compatible with the last release
run: python -m tools.contracts.interface_compat

standalone:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: "3.15"
- name: Install the build tools from the lock file
run: |
python -m pip install --quiet pip==26.3
python -m pip install --quiet -r pylock.toml
- name: The contracts module alone, outside the repository
run: python -m tools.ci.standalone contracts

standalone-store:
# The store alone (#486 §5 step 2): its wheel with only the contracts wheel, in a clean
# environment outside the repository, passing its own tests and its contract's kit, with
# the same disposable services its tests need in python.yml, so none is skipped.
runs-on: ubuntu-latest
timeout-minutes: 30
# PostgreSQL catalog for the store tests (P1-02); a disposable database, not a secret.
services:
postgres:
# postgres:18.6-trixie
image: postgres@sha256:edb0c08dc33befd0262ab3b5cf1c4f3f8cfd8b09e88c05b45adee29d6761ddb6
env:
POSTGRES_PASSWORD: ci-disposable-database
ports: ["5432:5432"]
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 2s --health-timeout 5s --health-retries 30
# ClickHouse for the Direct Query tests (P2-31, #225); a disposable server, not a secret.
# The tests create their own read-only profiles, users and synthetic tables.
clickhouse:
# clickhouse/clickhouse-server:26.10.2.103 (#429: the latest stable)
image: clickhouse/clickhouse-server@sha256:b2bfa4d8399ee9973868cc5c1754cb506a6f4b86fbddc41010cc090beb55c85e
env:
CLICKHOUSE_USER: ci
CLICKHOUSE_PASSWORD: ci-disposable-database
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
ports: ["8123:8123"]
options: >-
--health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'"
--health-interval 2s --health-timeout 5s --health-retries 60
# A ClickHouse of a version the store does not test, for the store kit's
# ENGINE_VERSION_UNTESTED case (#486); a disposable server, not a secret.
clickhouse-untested:
# clickhouse/clickhouse-server:26.9.25.15 (between the tested 26.8 and 26.10)
image: clickhouse/clickhouse-server@sha256:19bfacb135479a892551a8155cfbbc8211aedf2aa888cd91950b9ee7e76cc7e3
env:
CLICKHOUSE_USER: ci
CLICKHOUSE_PASSWORD: ci-disposable-database
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
ports: ["8124:8123"]
options: >-
--health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'"
--health-interval 2s --health-timeout 5s --health-retries 60
env:
STORE_TEST_POSTGRES: host=localhost port=5432 dbname=postgres user=postgres
STORE_TEST_POSTGRES_PASSWORD: ci-disposable-database
STORE_TEST_CLICKHOUSE: localhost:8123:ci
STORE_TEST_CLICKHOUSE_PASSWORD: ci-disposable-database
# The PostgreSQL with TLS started below (#120), reached through 127.0.0.2: a host that is
# not the local machine for the catalog rule, so verify-full is required and tested.
STORE_TEST_POSTGRES_TLS: host=127.0.0.2 port=5433 dbname=postgres user=postgres
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Start a PostgreSQL with TLS for the catalog tests (#120) and Azurite (#127)
# Started before the Python install and waited for after it (#373): the Azurite image
# downloads while Python and the lock file install.
# A certificate authority and a server certificate made for this job only: disposable,
# never committed, not secrets. The CA file is given to the tests by its path.
run: |
dir="$RUNNER_TEMP/postgres-tls"
mkdir -p "$dir"
cd "$dir"
openssl req -x509 -newkey rsa:3072 -nodes -days 2 -subj "/CN=ci-test-ca" \
-addext "basicConstraints=critical,CA:TRUE" \
-addext "keyUsage=critical,keyCertSign,cRLSign" \
-keyout ca.key -out ca.crt
openssl req -newkey rsa:3072 -nodes -subj "/CN=localhost" \
-keyout server.key -out server.csr
printf '%s\n' "basicConstraints=CA:FALSE" \
"keyUsage=critical,digitalSignature,keyEncipherment" \
"extendedKeyUsage=serverAuth" \
"subjectAltName=DNS:localhost,IP:127.0.0.1,IP:127.0.0.2" > server.ext
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 2 \
-extfile server.ext -out server.crt
rm ca.key server.csr
# The image's postgres user (uid 999) must own the key, readable by it only.
sudo chown 999:999 server.key server.crt
sudo chmod 600 server.key
# postgres:18.6-trixie, the same image as the service above.
docker run --detach --name postgres-tls --publish 5433:5432 \
--env POSTGRES_PASSWORD=ci-disposable-database \
--volume "$dir/server.crt:/tls/server.crt:ro" \
--volume "$dir/server.key:/tls/server.key:ro" \
postgres@sha256:edb0c08dc33befd0262ab3b5cf1c4f3f8cfd8b09e88c05b45adee29d6761ddb6 \
-c ssl=on -c ssl_cert_file=/tls/server.crt -c ssl_key_file=/tls/server.key
echo "STORE_TEST_POSTGRES_TLS_CA=$dir/ca.crt" >> "$GITHUB_ENV"
# azurite 3.37.0, blob service only, downloaded and started in the background; its
# output goes to a file the wait below shows when it fails.
nohup docker run --detach --name azurite --publish 10000:10000 \
mcr.microsoft.com/azure-storage/azurite@sha256:830430c1da1a2d537e08f3e6764dd1f5ae00cf0346bcaf625b968ec3f0971fd5 \
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck --loose \
> "$RUNNER_TEMP/azurite-start.log" 2>&1 &
- uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
with:
python-version: "3.15"
- name: Install the build tools from the lock file
run: |
python -m pip install --quiet pip==26.3
python -m pip install --quiet -r pylock.toml
- name: Install the store's DuckDB extensions, verified (#427)
run: python -m tools.ci.duckdb_extensions
- name: Wait for the PostgreSQL with TLS and Azurite
# The PostgreSQL is ready when it accepts TCP connections (the first start listens on a
# socket only). Azurite's development account is public and disposable, not a secret.
run: |
for attempt in $(seq 60); do
if docker exec postgres-tls pg_isready -U postgres -h 127.0.0.1 -q; then break; fi
sleep 1
done
docker exec postgres-tls pg_isready -U postgres -h 127.0.0.1
for attempt in $(seq 120); do
if curl --silent --output /dev/null http://127.0.0.1:10000/; then break; fi
sleep 1
done
if ! curl --silent --output /dev/null http://127.0.0.1:10000/; then
cat "$RUNNER_TEMP/azurite-start.log"
exit 1
fi
key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
echo "STORE_TEST_AZURE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=$key;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" >> "$GITHUB_ENV"
- name: The store module alone, outside the repository
run: python -m tools.ci.standalone store
2 changes: 1 addition & 1 deletion .github/workflows/property-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,6 @@ jobs:
env:
PROPERTY_PROFILE: nightly
run: |
python -m pytest -q -p tools.ci.timing --junitxml=junit.xml tests/properties
python -m pytest -q -p tools.ci.timing --junitxml=junit.xml tests/properties packages/store/tests/test_clickhouse_scanner_fuzzing.py
# Each test's time to this log (#375): what test-trends.yml reads back.
python -m tools.ci.durations emit junit.xml
15 changes: 15 additions & 0 deletions .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,19 @@ jobs:
options: >-
--health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'"
--health-interval 2s --health-timeout 5s --health-retries 60
# A ClickHouse of a version the store does not test, for the store kit's
# ENGINE_VERSION_UNTESTED case (#486); a disposable server, not a secret.
clickhouse-untested:
# clickhouse/clickhouse-server:26.9.25.15 (between the tested 26.8 and 26.10)
image: clickhouse/clickhouse-server@sha256:19bfacb135479a892551a8155cfbbc8211aedf2aa888cd91950b9ee7e76cc7e3
env:
CLICKHOUSE_USER: ci
CLICKHOUSE_PASSWORD: ci-disposable-database
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
ports: ["8124:8123"]
options: >-
--health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'"
--health-interval 2s --health-timeout 5s --health-retries 60
env:
STORE_TEST_POSTGRES: host=localhost port=5432 dbname=postgres user=postgres
STORE_TEST_POSTGRES_PASSWORD: ci-disposable-database
Expand Down Expand Up @@ -382,6 +395,8 @@ jobs:
python -m pip install --quiet pip==26.3
pins=$(python -m tools.ci.pins pylock.toml pytest pluggy iniconfig packaging pygments duckdb)
python -m pip install --quiet --only-binary=:all: --no-deps $pins colorama==0.4.6
# The shared test helpers these tests use (#486), from the checkout.
python -m pip install --quiet --no-deps -e packages/testing
- run: python -m pytest -p tools.ci.order tests/tools/test_fixtures.py tests/tools/test_order.py tests/tools/test_shard.py tests/tools/test_pins.py tests/tools/test_synthetic.py
env:
PYTEST_RANDOM_ORDER: ${{ github.run_id }}
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/scripts/ci_alerts.toml
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,10 @@ workflow = "ci-alerts-selftest.yml"
events = ["workflow_dispatch"]
required_jobs = ["selftest"]
cadence = "none"

[[watch]]
# Interface contracts (#486): the record and its compatibility, and the standalone module.
workflow = "interfaces.yml"
events = ["push"]
required_jobs = ["interface-contracts", "standalone"]
cadence = "push"
2 changes: 1 addition & 1 deletion benchmarks/source-stream.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Source isolation cost (#372, ADR-0031 §6)

Measured with `tools/benchmarks/source_stream.py` on 2026-11-07. Every number below comes from that run; none is estimated.
Measured with `tools/benchmarks/source_stream.py` (since #486, `packages/store/benchmarks/source_stream.py`) on 2026-11-07. Every number below comes from that run; none is estimated.

- **Code:** the #372 branch on top of `493426e` (main), with the script as committed here; the measured code is that of this pull request
- **Command:** `python -m tools.benchmarks.source_stream --rows 10000000 --rows 50000000 --runs 3`
Expand Down
11 changes: 7 additions & 4 deletions conformance/runners/corpus_clickhouse.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Running a corpus case on ClickHouse (P2-31 PR 4, #225; ADR-0027 §4, ADR-0028 §3): the case's
tables loaded into a ClickHouse database with the mapped types, and the case answered there
through the product's own wiring: the store's verified, read-only connection
(`orvex.store.clickhouse`), the server's adapter (`clickhouse_adapter`) and the evidence its
(its contract's `clickhouse`), the server's adapter (`clickhouse_adapter`) and the evidence its
schedule verifies (`clickhouse_evidence`).

The server is `CORPUS_CLICKHOUSE` (`host:port:user`, an administrator of a disposable server)
Expand Down Expand Up @@ -30,7 +30,11 @@
from pathlib import Path
from typing import Any

from orvex.contracts.store.client import Lake
from orvex.contracts.store.client_sources import connect_clickhouse
from orvex.contracts.store.interface import REQUIRED_SETTINGS
from orvex.contracts.store.v1 import ReadRequest, TablesRequest
from orvex.contracts.store.values import Budget, ClickHouseTarget
from orvex.semantic.adapters import clickhouse as ch
from orvex.semantic.adapters.clickhouse import CAPABILITIES as CLICKHOUSE_CAPABILITIES
from orvex.semantic.compiler.compiled import CompiledModel
Expand All @@ -42,8 +46,7 @@
from orvex.server.clickhouse_engines import tables
from orvex.server.clickhouse_evidence import EvidenceSchedule, checked
from orvex.server.identity import load_identity
from orvex.store.clickhouse import REQUIRED_SETTINGS, Budget, ClickHouse, ClickHouseTarget
from orvex.store.lake import Lake
from orvex.server.store_access import store

from conformance.runners import corpus_execution as ce

Expand Down Expand Up @@ -166,7 +169,7 @@ def clickhouse(
with database() as (admin, name):
load(admin, name, lake)
target = ClickHouseTarget(admin.host, admin.port, name, name, READER_PASSWORD, tls=False)
engine = ClickHouse.connect(target, BUDGET)
engine = connect_clickhouse(store(), target, BUDGET)
entry = EngineEntry(
CONNECTION,
"clickhouse",
Expand Down
13 changes: 7 additions & 6 deletions conformance/runners/corpus_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,15 @@
from typing import Any, cast

import yaml
from orvex.contracts.store.client import Lake
from orvex.contracts.store.errors import StoreError
from orvex.contracts.store.v1 import (
LocalLakeLocation,
Parameter,
ParameterType,
StatementRequest,
)
from orvex.contracts.store.values import Limits
from orvex.semantic.compiler.catalog import SourceColumn
from orvex.semantic.compiler.compiled import CompiledModel, compile_model
from orvex.semantic.errors import SemanticError
Expand All @@ -37,9 +40,7 @@
from orvex.semantic.policies import PolicyModel, read_policies
from orvex.semantic.query.binding import Limits as QueryLimits
from orvex.semantic.receipt import Key
from orvex.store.errors import StoreError
from orvex.store.json_encoding import arrow_to_json
from orvex.store.lake import Lake, Limits, open_lake
from orvex.server.store_access import store

from conformance.runners import bi_platform_adapter as adapter

Expand Down Expand Up @@ -167,7 +168,7 @@ def case_lake(
) -> Iterator[Lake]:
with tempfile.TemporaryDirectory() as folder:
location = LocalLakeLocation(kind="lake_location", catalog="local", path=folder)
lake = open_lake(location, Limits("1GB", 2, 2, 0, 600_000))
lake = store().open_lake(location, Limits("1GB", 2, 2, 0, 600_000))
try:
load_tables(lake, inputs, catalog, schema)
yield lake
Expand All @@ -179,7 +180,7 @@ def engine_for(lake: Lake, **change: Any) -> Engine: # noqa: ANN401
snapshot = lake.current_snapshot()
arguments: dict[str, Any] = {
"read": lake.read,
"encoder": arrow_to_json,
"encoder": store().arrow_to_json,
"snapshot": {"id": snapshot.id, "committed_at": snapshot.committed_at.isoformat()},
"engine_version": version("duckdb"),
"result_key": RESULT_KEY,
Expand Down Expand Up @@ -217,7 +218,7 @@ def viewer_context(

def rows_of(found: Answer) -> list[list[str | None]]:
"""The result's values as `store/v1` JSON text, positionally."""
document = json.loads(found.result.wire(arrow_to_json))
document = json.loads(found.result.wire(store().arrow_to_json))
return [[None if v is None else str(v) for v in row] for row in document["rows"]]


Expand Down
12 changes: 7 additions & 5 deletions conformance/runners/corpus_existing_ducklake.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from __future__ import annotations

import contextlib
import datetime as dt
import os
import shutil
Expand All @@ -31,13 +32,16 @@

import duckdb
import pyarrow as pa
from orvex.contracts.store.client import Lake
from orvex.contracts.store.client_sources import open_existing_lake
from orvex.contracts.store.v1 import (
ExistingLakeAttachment,
PostgresLakeLocation,
ReadRequest,
Snapshot,
TablesRequest,
)
from orvex.contracts.store.values import ExistingLakeLimits
from orvex.semantic.compiler.compiled import CompiledModel
from orvex.semantic.engines import EngineEntry, Routing
from orvex.semantic.execution import answer
Expand All @@ -48,9 +52,7 @@
from orvex.server.existing_lake_evidence import violated
from orvex.server.existing_lake_snapshots import SnapshotSchedule
from orvex.server.identity import load_identity
from orvex.store.existing_lake import ExistingLake
from orvex.store.existing_lake_reads import LakeReads
from orvex.store.lake import Lake
from orvex.server.store_access import store

from conformance.runners import corpus_execution as ce

Expand Down Expand Up @@ -214,8 +216,8 @@ def existing_ducklake(
EXISTING_DUCKLAKE_CAPABILITIES,
)
routing = Routing({str(d["name"]): entry for d in datasets})
with ExistingLake(made.attachment(), memory_limit="1GB", threads=2) as existing:
reads = LakeReads(existing, timeout_ms=TIMEOUT_MS)
limits = ExistingLakeLimits(memory_limit="1GB", threads=2, timeout_ms=TIMEOUT_MS)
with contextlib.closing(open_existing_lake(store(), made.attachment(), limits)) as reads:

def evidence(snapshot: Snapshot) -> str | None:
return violated(
Expand Down
Loading
Loading