From f75ae87a95996d44ea24796623762c81a193c1a2 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 14 Nov 2026 21:34:48 -0400 Subject: [PATCH 01/21] test(contracts): the case matrix of the interface-contract foundations The format, kits, bindings and manifest; the assembly check; the consumer boundaries; contract compatibility; the standalone job. Written before the code: every test fails without it (#486 step 1). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- packages/contracts/tests/sample_contract.py | 131 ++++++++ packages/contracts/tests/test_binding.py | 228 ++++++++++++++ packages/contracts/tests/test_interface.py | 257 +++++++++++++++ packages/contracts/tests/test_kit.py | 127 ++++++++ packages/contracts/tests/test_manifest.py | 89 ++++++ packages/contracts/tests/test_registry.py | 98 ++++++ packages/server/tests/test_assembly.py | 216 +++++++++++++ .../contracts}/test_http_api_v1.py | 2 +- tests/tools/test_consumer_boundaries.py | 115 +++++++ tests/tools/test_interface_compat.py | 298 ++++++++++++++++++ tests/tools/test_standalone.py | 130 ++++++++ 11 files changed, 1690 insertions(+), 1 deletion(-) create mode 100644 packages/contracts/tests/sample_contract.py create mode 100644 packages/contracts/tests/test_binding.py create mode 100644 packages/contracts/tests/test_interface.py create mode 100644 packages/contracts/tests/test_kit.py create mode 100644 packages/contracts/tests/test_manifest.py create mode 100644 packages/contracts/tests/test_registry.py create mode 100644 packages/server/tests/test_assembly.py rename {packages/contracts/tests => tests/contracts}/test_http_api_v1.py (97%) create mode 100644 tests/tools/test_consumer_boundaries.py create mode 100644 tests/tools/test_interface_compat.py create mode 100644 tests/tools/test_standalone.py diff --git a/packages/contracts/tests/sample_contract.py b/packages/contracts/tests/sample_contract.py new file mode 100644 index 00000000..b0943eb7 --- /dev/null +++ b/packages/contracts/tests/sample_contract.py @@ -0,0 +1,131 @@ +"""A small interface contract, an implementation and a kit for the tests of the format itself.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +from typing import Protocol +from zoneinfo import ZoneInfo + +from orvex.contracts.binding import Bound +from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.kit import Kit, KitCase + + +class LedgerError(Exception): + """The ledger's one error type: a stable code, a message and details.""" + + def __init__(self, code: str, message: str, details: dict[str, str]) -> None: + super().__init__(f"{code}: {message}") + self.code = code + self.message = message + self.details = details + + +@dataclass(frozen=True, slots=True) +class Reading: + name: str + amount: Decimal + at: datetime + + +class Ledger(Protocol): + def put(self, reading: Reading) -> None: ... + + def get(self, name: str) -> Reading | None: ... + + def names(self) -> tuple[str, ...]: ... + + +LEDGER = InterfaceContract( + name="ledger", + version=Version(1, 0), + protocol=Ledger, + error_type=LedgerError, + capabilities=("read", "write"), + errors=("LEDGER_NAME_EMPTY",), + operations=( + Operation("put", "write", ("LEDGER_NAME_EMPTY",)), + Operation("get", "read"), + Operation("names", "read"), + ), +) + + +class MemoryLedger: + """A correct implementation of the ledger.""" + + def __init__(self) -> None: + self._readings: dict[str, Reading] = {} + + def put(self, reading: Reading) -> None: + if not reading.name: + raise LedgerError("LEDGER_NAME_EMPTY", "a reading needs a name", {"field": "name"}) + self._readings[reading.name] = reading + + def get(self, name: str) -> Reading | None: + return self._readings.get(name) + + def names(self) -> tuple[str, ...]: + return tuple(sorted(self._readings)) + + +READING = Reading( + "meter-7", Decimal("12.50"), datetime(2026, 3, 8, 1, 30, tzinfo=ZoneInfo("America/La_Paz")) +) + + +def _expect(condition: bool, text: str) -> None: + if not condition: + raise AssertionError(text) + + +def round_trip(ledger: Bound) -> None: + ledger.put(READING) + found = ledger.get(READING.name) + _expect(found == READING, f"read back {found!r}") + _expect(isinstance(found, Reading) and str(found.amount) == "12.50", "the scale changed") + _expect(isinstance(found, Reading) and found.at.tzinfo == READING.at.tzinfo, "the zone changed") + + +def missing(ledger: Bound) -> None: + _expect(ledger.get("absent") is None, "a missing name is not None") + + +def empty_name_refused(ledger: Bound) -> None: + try: + ledger.put(Reading("", Decimal(1), READING.at)) + except LedgerError as error: + _expect(error.code == "LEDGER_NAME_EMPTY", f"code {error.code}") + _expect(error.details == {"field": "name"}, f"details {error.details}") + return + raise AssertionError("an empty name was accepted") + + +def names_sorted(ledger: Bound) -> None: + for name in ("b", "a"): + ledger.put(Reading(name, Decimal(0), READING.at)) + _expect(ledger.names() == ("a", "b"), "names are not sorted") + + +LEDGER_KIT = Kit( + contract=LEDGER, + recorded=4, + cases=( + KitCase( + "put.round-trip-exact", + "put", + "a reading read back keeps its name, amount scale and time zone", + round_trip, + ), + KitCase("get.missing-is-none", "get", "an unknown name reads as None", missing), + KitCase( + "put.empty-name-refused", + "put", + "an empty name is refused with LEDGER_NAME_EMPTY", + empty_name_refused, + ), + KitCase("names.sorted", "names", "names come back sorted", names_sorted), + ), +) diff --git a/packages/contracts/tests/test_binding.py b/packages/contracts/tests/test_binding.py new file mode 100644 index 00000000..759d8c88 --- /dev/null +++ b/packages/contracts/tests/test_binding.py @@ -0,0 +1,228 @@ +"""The bindings and their codec (#486 G5): values cross serialized exactly, live objects never.""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass +from datetime import UTC, date, datetime, time, timedelta, timezone +from decimal import Decimal +from enum import Enum +from zoneinfo import ZoneInfo + +import pytest +from orvex.contracts.binding import Bound +from orvex.contracts.codec import Codec +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.kit_run import run_kit, verify_kit +from pydantic import BaseModel + +from .sample_contract import LEDGER, LEDGER_KIT, READING, LedgerError, MemoryLedger, Reading + + +class Color(Enum): + RED = "red" + + +class Shape(BaseModel): + side: Decimal + + +@dataclass(frozen=True, slots=True) +class Undeclared: + value: int + + +@dataclass(frozen=True, slots=True) +class ReadingChild(Reading): + pass + + +CODEC = Codec(frozenset({Reading, Color, Shape})) + +VALUES: list[object] = [ + None, + True, + False, + 0, + -(2**127), + 2**128 - 1, + 1.5, + float("inf"), + -0.0, + "", + "ñandú ✓", + b"\x00\xff", + Decimal("12.50"), + Decimal("-0.000"), + Decimal("123456789012345678901234567890.123456789"), + datetime(2026, 3, 8, 1, 30, tzinfo=ZoneInfo("America/La_Paz")), + datetime(2026, 11, 1, 1, 30, fold=1, tzinfo=ZoneInfo("America/New_York")), + datetime(2026, 3, 8, 1, 30, 0, 123456), + datetime(2026, 3, 8, tzinfo=UTC), + datetime(2026, 3, 8, tzinfo=timezone(timedelta(hours=-4))), + date(2026, 2, 28), + time(23, 59, 59, 999999), + time(1, 2, tzinfo=timezone(timedelta(hours=5, minutes=30))), + timedelta(days=-1, seconds=5, microseconds=7), + (1, "a", (Decimal("1.0"),)), + [1, [2, None]], + {"a": {"b": [Decimal("0.10")]}, "": None}, + Color.RED, + Shape(side=Decimal("2.50")), + READING, + (), + [], + {}, +] + + +def same(left: object, right: object) -> bool: + """Equal, with the same type all the way down (a tuple never comes back a list).""" + if type(left) is not type(right): + return False + if isinstance(left, (tuple, list)): + assert isinstance(right, (tuple, list)) + return len(left) == len(right) and all(map(same, left, right, strict=True)) + if isinstance(left, dict): + assert isinstance(right, dict) + return left.keys() == right.keys() and all(same(left[k], right[k]) for k in left) + if isinstance(left, Decimal): + return str(left) == str(right) + if isinstance(left, (datetime, time)): + assert isinstance(right, (datetime, time)) + return ( + left == right + and str(left.tzinfo) == str(right.tzinfo) + and left.fold == right.fold + and left.isoformat() == right.isoformat() + ) + return left == right or (left != left and right != right) + + +class TestCodec: + @pytest.mark.parametrize("value", VALUES, ids=repr) + def test_every_value_comes_back_exactly(self, value: object) -> None: + back = CODEC.decode(CODEC.encode(value)) + assert same(value, back), (value, back) + + def test_a_zone_comes_back_as_the_same_zone(self) -> None: + back = CODEC.decode(CODEC.encode(READING.at)) + assert isinstance(back, datetime) + assert back.tzinfo == ZoneInfo("America/La_Paz") + + @pytest.mark.parametrize( + "value", + [ + threading.Lock(), + iter([1]), + lambda: 1, + object(), + Undeclared(1), + ReadingChild("x", Decimal(1), READING.at), + {1: "a"}, + {"a": threading.Lock()}, + {1, 2}, + bytearray(b"x"), + time(1, tzinfo=ZoneInfo("Europe/Madrid")), + ], + ids=lambda value: type(value).__name__, + ) + def test_a_live_or_undeclared_value_is_refused(self, value: object) -> None: + with pytest.raises(InterfaceContractError) as raised: + CODEC.encode(value) + assert raised.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" + + @pytest.mark.parametrize( + "data", + [b"not json", b'{"$unknown": 1}', b'{"$dataclass": "nowhere.Thing", "fields": {}}', b"{}"], + ) + def test_bytes_that_are_not_an_encoded_value_are_refused(self, data: bytes) -> None: + with pytest.raises(InterfaceContractError) as raised: + CODEC.decode(data) + assert raised.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" + + +class LiveNames(tuple[str, ...]): + """A live view of the names: equal to the tuple in process, but it holds a lock.""" + + lock = threading.Lock() + + +class LiveLedger(MemoryLedger): + """A planted violation: `names` hands out a live object instead of a value.""" + + def names(self) -> tuple[str, ...]: + return LiveNames(sorted(self._readings)) + + +class WrongCode(MemoryLedger): + def get(self, name: str) -> Reading | None: + raise LedgerError("LEDGER_GONE", "not declared for get", {}) + + +class TestBound: + @pytest.mark.parametrize("serialized", [False, True]) + def test_only_the_contract_s_operations_are_reachable(self, serialized: bool) -> None: + bound = Bound(LEDGER, MemoryLedger(), serialized=serialized) + assert bound.serialized is serialized + with pytest.raises(AttributeError): + bound.__getattr__("_readings") + with pytest.raises(AttributeError): + bound.__getattr__("drop") + + @pytest.mark.parametrize("serialized", [False, True]) + def test_an_implementation_without_the_operation_is_refused(self, serialized: bool) -> None: + class Partial: + def get(self, name: str) -> None: + return None + + bound = Bound(LEDGER, Partial(), serialized=serialized) + with pytest.raises(InterfaceContractError) as raised: + bound.names() + assert raised.value.code == "INTERFACE_OPERATION_MISSING" + + def test_serialized_calls_cross_as_copies(self) -> None: + ledger = MemoryLedger() + Bound(LEDGER, ledger, serialized=True).put(READING) + kept = ledger.get(READING.name) + assert kept == READING + assert kept is not READING + assert Bound(LEDGER, ledger, serialized=True).get(READING.name) is not kept + in_process = MemoryLedger() + Bound(LEDGER, in_process, serialized=False).put(READING) + assert in_process.get(READING.name) is READING + + def test_a_declared_error_crosses_with_its_code_message_and_details(self) -> None: + with pytest.raises(LedgerError) as raised: + Bound(LEDGER, MemoryLedger(), serialized=True).put(Reading("", Decimal(1), READING.at)) + assert raised.value.code == "LEDGER_NAME_EMPTY" + assert raised.value.message == "a reading needs a name" + assert raised.value.details == {"field": "name"} + assert raised.value.__cause__ is None + + @pytest.mark.parametrize("serialized", [False, True]) + def test_an_undeclared_error_code_is_refused_in_both_bindings(self, serialized: bool) -> None: + with pytest.raises(InterfaceContractError) as raised: + Bound(LEDGER, WrongCode(), serialized=serialized).get("a") + assert raised.value.code == "INTERFACE_ERROR_UNDECLARED" + + def test_a_live_result_is_refused_when_serialized(self) -> None: + ledger = LiveLedger() + assert Bound(LEDGER, ledger, serialized=False).names() == () + with pytest.raises(InterfaceContractError) as raised: + Bound(LEDGER, ledger, serialized=True).names() + assert raised.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" + + +class TestPlantedLiveObject: + """G5: a planted operation that returns a live object fails the serializing run.""" + + def test_the_kit_fails_the_serialized_run_and_the_runs_differ(self) -> None: + serialized = run_kit(LEDGER_KIT, LiveLedger, serialized=True) + failed = [result.id for result in serialized.results if not result.passed] + assert failed == ["names.sorted"] + with pytest.raises(InterfaceContractError) as raised: + verify_kit(LEDGER_KIT, LiveLedger) + assert raised.value.code == "KIT_FAILED" + assert "KIT_BINDINGS_DIFFER" in raised.value.message + assert "names.sorted (serialized)" in raised.value.message diff --git a/packages/contracts/tests/test_interface.py b/packages/contracts/tests/test_interface.py new file mode 100644 index 00000000..e3cf4a70 --- /dev/null +++ b/packages/contracts/tests/test_interface.py @@ -0,0 +1,257 @@ +"""The interface-contract format: declaration, validation and description (#486 §5 step 1).""" + +from __future__ import annotations + +import dataclasses +import threading +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from enum import Enum +from typing import Any, Literal, Protocol + +import pytest +from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.interface_errors import InterfaceContractError +from pydantic import BaseModel + +from .sample_contract import LEDGER, LedgerError, Reading + +READING_TYPE = f"{Reading.__module__}.{Reading.__qualname__}" + + +class Color(Enum): + RED = "red" + + +class Shape(BaseModel): + side: Decimal + + +@dataclass(frozen=True, slots=True) +class Inner: + when: datetime + + +@dataclass(frozen=True, slots=True) +class Outer: + inner: Inner + items: tuple[Inner, ...] + + +@dataclass(slots=True) +class Mutable: + value: int + + +def contract_for(method: Callable[..., object], **changes: object) -> InterfaceContract: + protocol = type("Probe", (Protocol,), {"use": method}) + fields: dict[str, Any] = { + "name": "probe", + "version": Version(1, 0), + "protocol": protocol, + "error_type": LedgerError, + "capabilities": ("use",), + "errors": (), + "operations": (Operation("use", "use"),), + } + fields.update(changes) + return InterfaceContract(**fields) + + +def code_of(build: Callable[[], object]) -> str: + with pytest.raises(InterfaceContractError) as raised: + build() + return raised.value.code + + +class TestVersion: + @pytest.mark.parametrize( + ("text", "expected"), [("1.0", Version(1, 0)), ("2.13", Version(2, 13))] + ) + def test_parses_major_dot_minor(self, text: str, expected: Version) -> None: + assert Version.parse(text) == expected + assert str(expected) == text + + @pytest.mark.parametrize("text", ["1", "01.0", "1.0.0", "0.1", "v1.0", "1.01", ""]) + def test_anything_else_is_refused(self, text: str) -> None: + assert code_of(lambda: Version.parse(text)) == "INTERFACE_VERSION_INVALID" + + def test_a_version_provides_its_major_at_a_lower_or_equal_minor(self) -> None: + assert Version(1, 3).provides(Version(1, 0)) + assert Version(1, 3).provides(Version(1, 3)) + assert not Version(1, 3).provides(Version(1, 4)) + assert not Version(2, 0).provides(Version(1, 0)) + assert not Version(1, 0).provides(Version(2, 0)) + + +class TestDescription: + def test_the_description_names_every_part_of_the_contract(self) -> None: + described = LEDGER.describe() + assert described["name"] == "ledger" + assert described["version"] == "1.0" + assert described["capabilities"] == ["read", "write"] + assert described["errors"] == ["LEDGER_NAME_EMPTY"] + operations = described["operations"] + assert isinstance(operations, dict) + assert sorted(operations) == ["get", "names", "put"] + assert operations["put"] == { + "capability": "write", + "errors": ["LEDGER_NAME_EMPTY"], + "parameters": [ + { + "name": "reading", + "kind": "positional-or-keyword", + "type": READING_TYPE, + "default": False, + } + ], + "returns": "None", + } + assert operations["get"]["returns"] == f"{READING_TYPE} | None" + assert operations["names"]["returns"] == "tuple[str, ...]" + + def test_operations_are_found_by_name_and_by_capability(self) -> None: + assert LEDGER.operation("get").capability == "read" + assert [o.name for o in LEDGER.operations_of("read")] == ["get", "names"] + assert code_of(lambda: LEDGER.operation("drop")) == "INTERFACE_OPERATION_UNKNOWN" + + +def _plain(self: object, value: int) -> None: ... + + +class TestRefusals: + def test_a_valid_probe_is_accepted(self) -> None: + assert contract_for(_plain).operation("use").name == "use" + + @pytest.mark.parametrize( + "changes", + [ + {"name": "Probe"}, + {"name": ""}, + {"operations": (Operation("use", "other"),)}, + {"capabilities": ("use", "idle")}, + {"capabilities": ("use", "use")}, + {"operations": (Operation("use", "use", ("PROBE_FAILED",)),)}, + {"errors": ("probe failed",)}, + {"errors": ("PROBE_FAILED", "PROBE_FAILED")}, + {"operations": (Operation("use", "use"), Operation("use", "use"))}, + {"operations": (Operation("use", "use"), Operation("other", "use"))}, + {"operations": ()}, + {"error_type": int}, + ], + ids=lambda changes: ",".join(changes), + ) + def test_an_inconsistent_declaration_is_refused(self, changes: dict[str, object]) -> None: + assert code_of(lambda: contract_for(_plain, **changes)) == "INTERFACE_CONTRACT_INVALID" + + def test_a_protocol_method_that_is_not_declared_is_refused(self) -> None: + protocol = type("Probe", (Protocol,), {"use": _plain, "hidden": _plain}) + assert code_of(lambda: contract_for(_plain, protocol=protocol)) == ( + "INTERFACE_CONTRACT_INVALID" + ) + + @pytest.mark.parametrize("where", ["contract", "operation"]) + def test_unsupported_operation_can_never_be_declared(self, where: str) -> None: + unsupported = ("UNSUPPORTED_OPERATION",) + changes: dict[str, object] = {"errors": unsupported} + if where == "operation": + changes["operations"] = (Operation("use", "use", unsupported),) + assert code_of(lambda: contract_for(_plain, **changes)) == ( + "INTERFACE_UNSUPPORTED_OPERATION_DECLARED" + ) + + def test_unannotated_and_variadic_parameters_are_refused(self) -> None: + def unannotated(self: object, value): # type: ignore[no-untyped-def] # noqa: ANN001, ANN202 + return value + + def variadic(self: object, *values: int) -> None: ... + + def keywords(self: object, **values: int) -> None: ... + + for method in (unannotated, variadic, keywords): + with pytest.raises(InterfaceContractError) as raised: + contract_for(method) + assert raised.value.code == "INTERFACE_CONTRACT_INVALID" + + +def _takes(annotation: object) -> Callable[..., object]: + def method(self: object, value: Any) -> None: ... # noqa: ANN401 - replaced below + + method.__annotations__ = {"value": annotation, "return": type(None)} + return method + + +def _returns(annotation: object) -> Callable[..., object]: + def method(self: object) -> None: ... + + method.__annotations__ = {"return": annotation} + return method + + +SERIALIZABLE: list[object] = [ + int, + bool, + float, + str, + bytes, + Decimal, + datetime, + date, + time, + timedelta, + type(None), + int | None, + Literal["a", 1, True, None], + tuple[int, ...], + tuple[str, Decimal], + list[Decimal], + dict[str, list[int]], + Color, + Shape, + Outer, +] + +LIVE: list[object] = [ + Any, + object, + Callable[[], int], + Iterator[int], + tuple, + list, + dict, + dict[int, str], + tuple[()], + threading.Lock, + Mutable, + Literal[Decimal(1)], + list[Callable[[], int]], + int | Iterator[int], +] + + +class TestCrossingTypes: + @pytest.mark.parametrize("annotation", SERIALIZABLE, ids=repr) + def test_plain_values_may_cross(self, annotation: object) -> None: + contract_for(_takes(annotation)) + contract_for(_returns(annotation)) + + @pytest.mark.parametrize("annotation", LIVE, ids=repr) + def test_a_live_object_may_never_cross(self, annotation: object) -> None: + assert ( + code_of(lambda: contract_for(_takes(annotation))) == "INTERFACE_TYPE_NOT_SERIALIZABLE" + ) + assert code_of(lambda: contract_for(_returns(annotation))) == ( + "INTERFACE_TYPE_NOT_SERIALIZABLE" + ) + + def test_the_classes_that_cross_are_collected(self) -> None: + assert contract_for(_takes(Outer)).value_classes() == frozenset({Outer, Inner}) + assert contract_for(_returns(list[Color | Shape])).value_classes() == {Color, Shape} + assert LEDGER.value_classes() == {Reading} + + def test_replace_validates_again(self) -> None: + assert code_of(lambda: dataclasses.replace(LEDGER, name="Bad")) == ( + "INTERFACE_CONTRACT_INVALID" + ) diff --git a/packages/contracts/tests/test_kit.py b/packages/contracts/tests/test_kit.py new file mode 100644 index 00000000..94ee5c09 --- /dev/null +++ b/packages/contracts/tests/test_kit.py @@ -0,0 +1,127 @@ +"""The conformance-kit format and its run (#486 G1): every case, a recorded count, no skips.""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Callable + +import pytest +from orvex.contracts.binding import Bound +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.kit import Kit, KitCase +from orvex.contracts.kit_run import kit_problems, run_kit, verify_kit + +from .sample_contract import LEDGER, LEDGER_KIT, MemoryLedger, Reading + + +def code_of(build: Callable[[], object]) -> str: + with pytest.raises(InterfaceContractError) as raised: + build() + return raised.value.code + + +class RoundingLedger(MemoryLedger): + """A planted implementation that fails one case: it drops the amount's scale.""" + + def put(self, reading: Reading) -> None: + super().put(dataclasses.replace(reading, amount=reading.amount.normalize())) + + +def test_a_correct_implementation_passes_every_case_in_both_bindings() -> None: + in_process, serialized = verify_kit(LEDGER_KIT, MemoryLedger) + assert not in_process.serialized + assert serialized.serialized + for report in (in_process, serialized): + assert report.contract == "ledger" + assert [r.id for r in report.results] == [c.id for c in LEDGER_KIT.cases] + assert all(r.passed for r in report.results) + + +def test_a_planted_implementation_failing_one_case_fails_the_kit() -> None: + report = run_kit(LEDGER_KIT, RoundingLedger, serialized=False) + assert [r.id for r in report.results if not r.passed] == ["put.round-trip-exact"] + with pytest.raises(InterfaceContractError) as raised: + verify_kit(LEDGER_KIT, RoundingLedger) + assert raised.value.code == "KIT_FAILED" + assert "KIT_CASE_FAILED: put.round-trip-exact (in process)" in raised.value.message + assert "KIT_CASE_FAILED: put.round-trip-exact (serialized)" in raised.value.message + + +def test_each_case_gets_a_fresh_implementation() -> None: + made: list[MemoryLedger] = [] + + def make() -> MemoryLedger: + made.append(MemoryLedger()) + return made[-1] + + run_kit(LEDGER_KIT, make, serialized=False) + assert len(made) == LEDGER_KIT.recorded + + +def test_a_run_with_one_case_fewer_fails() -> None: + report = run_kit(LEDGER_KIT, MemoryLedger, serialized=False) + shorter = dataclasses.replace(report, results=report.results[:-1]) + problems = kit_problems(LEDGER_KIT, shorter) + assert problems == ["KIT_RUN_INCOMPLETE: names.sorted did not run (in process)"] + + +def test_a_skip_is_a_failure_since_the_kit_has_no_skips() -> None: + def skipping(ledger: Bound) -> None: + pytest.skip("not today") + + cases = (dataclasses.replace(LEDGER_KIT.cases[0], check=skipping), *LEDGER_KIT.cases[1:]) + kit = dataclasses.replace(LEDGER_KIT, cases=cases) + report = run_kit(kit, MemoryLedger, serialized=False) + assert [r.passed for r in report.results] == [False, True, True, True] + assert "not today" in report.results[0].detail + + +def test_the_description_lists_every_case_and_the_recorded_count() -> None: + assert LEDGER_KIT.describe() == { + "recorded": 4, + "cases": { + case.id: {"operation": case.operation, "expectation": case.expectation} + for case in LEDGER_KIT.cases + }, + } + + +def _noop(ledger: Bound) -> None: + ledger.get("x") + + +CASE = KitCase("get.extra", "get", "an extra case", _noop) + + +class TestDeclarationRefusals: + def test_a_recorded_count_that_differs_from_the_cases_is_refused(self) -> None: + assert code_of(lambda: dataclasses.replace(LEDGER_KIT, recorded=3)) == ( + "KIT_CASE_COUNT_CHANGED" + ) + assert code_of(lambda: dataclasses.replace(LEDGER_KIT, recorded=5)) == ( + "KIT_CASE_COUNT_CHANGED" + ) + + def test_an_operation_without_a_case_is_refused(self) -> None: + cases = tuple(case for case in LEDGER_KIT.cases if case.operation != "names") + assert code_of(lambda: Kit(LEDGER, recorded=len(cases), cases=cases)) == ( + "KIT_OPERATION_UNCOVERED" + ) + + @pytest.mark.parametrize( + "case", + [ + dataclasses.replace(CASE, id="put.round-trip-exact"), + dataclasses.replace(CASE, id="Bad Id"), + dataclasses.replace(CASE, operation="drop"), + dataclasses.replace(CASE, expectation=" "), + ], + ids=["duplicate", "id", "operation", "expectation"], + ) + def test_an_invalid_case_is_refused(self, case: KitCase) -> None: + cases = (*LEDGER_KIT.cases, case) + assert code_of(lambda: Kit(LEDGER, recorded=len(cases), cases=cases)) == "KIT_INVALID" + + def test_an_extra_valid_case_is_accepted(self) -> None: + cases = (*LEDGER_KIT.cases, CASE) + assert Kit(LEDGER, recorded=5, cases=cases).recorded == 5 diff --git a/packages/contracts/tests/test_manifest.py b/packages/contracts/tests/test_manifest.py new file mode 100644 index 00000000..6910a34a --- /dev/null +++ b/packages/contracts/tests/test_manifest.py @@ -0,0 +1,89 @@ +"""The module manifest `module.toml` (#486 §2.5): provides, requires and declared dependencies.""" + +from __future__ import annotations + +from importlib.resources import files + +import orvex.contracts +import pytest +from orvex.contracts.interface import Version +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.manifest import Provided, Required, parse_manifest + +FULL = """ +module = "store" + +[dependencies] +runtime = ["duckdb", "PyArrow", "orvex-contracts"] +tests = ["pytest"] + +[[provides]] +contract = "store" +version = "1.2" +capabilities = ["read", "write"] +implementation = "Store" + +[[requires]] +contract = "security-context" +version = "1.0" +capabilities = ["read"] +""" + + +def test_a_full_manifest_is_read() -> None: + manifest = parse_manifest(FULL, "store") + assert manifest.module == "store" + assert manifest.provides == (Provided("store", Version(1, 2), ("read", "write"), "Store"),) + assert manifest.requires == (Required("security-context", Version(1, 0), ("read",)),) + assert manifest.runtime == ("duckdb", "orvex-contracts", "pyarrow") + assert manifest.tests == ("pytest",) + + +def test_the_shipped_manifest_of_the_contracts_distribution_is_valid() -> None: + text = files(orvex.contracts).joinpath("module.toml").read_text("utf-8") + manifest = parse_manifest(text, "contracts") + assert manifest.module == "contracts" + assert manifest.provides == () + assert manifest.requires == () + assert manifest.runtime == ("pydantic",) + + +def test_provides_and_requires_may_be_absent() -> None: + manifest = parse_manifest('module = "mcp"\n[dependencies]\nruntime = []\ntests = []\n', "mcp") + assert (manifest.provides, manifest.requires, manifest.runtime) == ((), (), ()) + + +INVALID = { + "not toml": "module = ", + "no module": "[dependencies]\nruntime = []\ntests = []\n", + "bad module": 'module = "Store"\n[dependencies]\nruntime = []\ntests = []\n', + "unknown key": FULL + 'extra = "x"\n', + "unknown provides key": FULL.replace( + 'implementation = "Store"', 'implementation = "Store"\nskip = ["read"]' + ), + "no dependencies": 'module = "store"\n', + "unknown dependency kind": FULL.replace('tests = ["pytest"]', 'tests = ["pytest"]\ndev = []'), + "bad distribution": FULL.replace('"duckdb"', '"duck db"'), + "duplicated distribution": FULL.replace('"duckdb"', '"duckdb", "DuckDB"'), + "bad version": FULL.replace('"1.2"', '"1"'), + "no capabilities": FULL.replace('["read", "write"]', "[]"), + "duplicated capability": FULL.replace('["read", "write"]', '["read", "read"]'), + "bad capability": FULL.replace('["read", "write"]', '["Read"]'), + "dotted implementation": FULL.replace('"Store"', '"internal.Store"'), + "module path implementation": FULL.replace('"Store"', '"orvex.store.lake:Store"'), + "provides twice": FULL + + '[[provides]]\ncontract = "store"\nversion = "1.0"\ncapabilities = ["read"]\n' + + 'implementation = "Other"\n', + "requires twice": FULL + + '[[requires]]\ncontract = "security-context"\nversion = "1.0"\ncapabilities = ["read"]\n', + "bad contract": FULL.replace('contract = "store"', 'contract = "Store"'), + "not a string list": FULL.replace('tests = ["pytest"]', "tests = [1]"), +} + + +@pytest.mark.parametrize("text", INVALID.values(), ids=INVALID.keys()) +def test_an_invalid_manifest_is_refused(text: str) -> None: + with pytest.raises(InterfaceContractError) as raised: + parse_manifest(text, "store") + assert raised.value.code == "MODULE_MANIFEST_INVALID" + assert raised.value.details["module"] == "store" diff --git a/packages/contracts/tests/test_registry.py b/packages/contracts/tests/test_registry.py new file mode 100644 index 00000000..6534ce90 --- /dev/null +++ b/packages/contracts/tests/test_registry.py @@ -0,0 +1,98 @@ +"""Finding the interface contracts and kits a contracts package ships, by their place.""" + +from __future__ import annotations + +import importlib +import sys +from collections.abc import Iterator +from pathlib import Path + +import orvex.contracts +import pytest +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.registry import interface_contracts, kits + +CONTRACT = """ +from typing import Protocol +from orvex.contracts.interface import InterfaceContract, Operation, Version + +class Error(Exception): + def __init__(self, code: str, message: str, details: dict[str, str]) -> None: + super().__init__(code) + +class Clock(Protocol): + def now(self) -> int: ... + +{name} = InterfaceContract( + name="{contract}", version=Version(1, 0), protocol=Clock, error_type=Error, + capabilities=("read",), errors=(), operations=(Operation("now", "read"),), +) +""" + +KIT = """ +from orvex.contracts.kit import Kit, KitCase +from .interface import CLOCK + +def ticks(clock) -> None: + clock.now() + +CLOCK_KIT = Kit(CLOCK, recorded=1, cases=(KitCase("now.ticks", "now", "it answers", ticks),)) +""" + + +@pytest.fixture +def package(tmp_path: Path, request: pytest.FixtureRequest) -> Iterator[Path]: + name = f"probe_contracts_{request.node.name.split('[')[0][-30:]}" + root = tmp_path / name + root.mkdir() + (root / "__init__.py").write_text("") + sys.path.insert(0, str(tmp_path)) + try: + yield root + finally: + sys.path.remove(str(tmp_path)) + for module in [m for m in sys.modules if m == name or m.startswith(f"{name}.")]: + del sys.modules[module] + + +def write(root: Path, files: dict[str, str]) -> None: + for relative, text in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +def test_contracts_and_kits_are_found_in_their_places(package: Path) -> None: + write( + package, + { + "clock/__init__.py": "", + "clock/interface.py": CONTRACT.format(name="CLOCK", contract="clock"), + "clock/kit.py": KIT, + "plain/__init__.py": "", + "plain/v1.py": "X = 1\n", + }, + ) + loaded = importlib.import_module(package.name) + assert list(interface_contracts(loaded)) == ["clock"] + assert list(kits(loaded)) == ["clock"] + assert kits(loaded)["clock"].contract is interface_contracts(loaded)["clock"] + + +def test_two_contracts_with_one_name_are_refused(package: Path) -> None: + write( + package, + { + "a/__init__.py": "", + "a/interface.py": CONTRACT.format(name="CLOCK", contract="clock"), + "b/__init__.py": "", + "b/interface.py": CONTRACT.format(name="OTHER", contract="clock"), + }, + ) + with pytest.raises(InterfaceContractError) as raised: + interface_contracts(importlib.import_module(package.name)) + assert raised.value.code == "INTERFACE_CONTRACT_INVALID" + + +def test_every_contract_the_distribution_ships_has_its_kit() -> None: + assert set(kits(orvex.contracts)) == set(interface_contracts(orvex.contracts)) diff --git a/packages/server/tests/test_assembly.py b/packages/server/tests/test_assembly.py new file mode 100644 index 00000000..5eb3e154 --- /dev/null +++ b/packages/server/tests/test_assembly.py @@ -0,0 +1,216 @@ +"""The assembly check (#486 G1): the server refuses to start when a chosen implementation lacks a +required capability or version.""" + +from __future__ import annotations + +import sys +import types +from collections.abc import Callable +from typing import Protocol + +import pytest +from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.manifest import Manifest, Provided, Required +from orvex.server import cli +from orvex.server.assembly import ( + AssemblyRefused, + assembly_problems, + check_assembly, + check_installed_assembly, + installed_manifests, + load_implementations, +) +from orvex.server.identity import load_identity + + +class CacheError(Exception): + def __init__(self, code: str, message: str, details: dict[str, str]) -> None: + super().__init__(f"{code}: {message}") + + +class Cache(Protocol): + def get(self, key: str) -> bytes | None: ... + + def put(self, key: str, value: bytes) -> None: ... + + def size(self) -> int: ... + + +CACHE = InterfaceContract( + name="cache", + version=Version(1, 1), + protocol=Cache, + error_type=CacheError, + capabilities=("read", "write"), + errors=(), + operations=(Operation("get", "read"), Operation("size", "read"), Operation("put", "write")), +) + + +class Whole: + def get(self, key: str) -> bytes | None: + return None + + def put(self, key: str, value: bytes) -> None: + return None + + def size(self) -> int: + return 0 + + +class WithoutPut: + """A planted alternative that omits one operation.""" + + def get(self, key: str) -> bytes | None: + return None + + def size(self) -> int: + return 0 + + +def provider( + version: str = "1.1", capabilities: tuple[str, ...] = ("read", "write"), module: str = "alpha" +) -> Manifest: + provided = Provided("cache", Version.parse(version), capabilities, "Cache") + return Manifest(module, (provided,), (), (), ()) + + +def consumer(version: str = "1.0", capabilities: tuple[str, ...] = ("read", "write")) -> Manifest: + required = Required("cache", Version.parse(version), capabilities) + return Manifest("server", (), (required,), (), ()) + + +def problems(*manifests: Manifest, implementation: object = None) -> list[str]: + chosen = { + (m.module, p.contract): Whole() if implementation is None else implementation + for m in manifests + for p in m.provides + } + return assembly_problems(manifests, {"cache": CACHE}, chosen) + + +def codes(found: list[str]) -> list[str]: + return [problem.split(":", 1)[0] for problem in found] + + +def test_a_complete_implementation_at_a_compatible_version_assembles() -> None: + assert problems(provider(), consumer()) == [] + assert problems(provider("1.1"), consumer("1.1", ("read",))) == [] + check_assembly((provider(), consumer()), {"cache": CACHE}, {("alpha", "cache"): Whole()}) + + +def test_a_planted_implementation_missing_a_required_capability_is_refused() -> None: + found = problems(provider(capabilities=("read",)), consumer()) + assert codes(found) == ["ASSEMBLY_CAPABILITY_MISSING"] + assert "write" in found[0] + + +def test_a_planted_implementation_omitting_an_operation_is_refused() -> None: + found = problems(provider(), consumer(), implementation=WithoutPut()) + assert codes(found) == ["ASSEMBLY_OPERATION_MISSING"] + assert "put" in found[0] + + +def test_an_operation_that_is_not_callable_is_refused() -> None: + hollow = types.SimpleNamespace(get=None, put=Whole().put, size=Whole().size) + assert codes(problems(provider(), consumer(), implementation=hollow)) == [ + "ASSEMBLY_OPERATION_MISSING" + ] + + +@pytest.mark.parametrize( + ("provided", "required"), + [("1.0", "1.1"), ("1.1", "2.0"), ("1.2", "1.0")], + ids=["older minor", "other major", "newer than the installed contract"], +) +def test_an_incompatible_version_is_refused(provided: str, required: str) -> None: + assert "ASSEMBLY_VERSION_INCOMPATIBLE" in codes( + problems(provider(provided), consumer(required)) + ) + + +def test_no_implementation_and_two_implementations_are_refused() -> None: + assert codes(problems(consumer())) == ["ASSEMBLY_NO_IMPLEMENTATION"] + assert codes(problems(provider(), provider(module="beta"), consumer())) == [ + "ASSEMBLY_AMBIGUOUS_IMPLEMENTATION" + ] + + +def test_an_unknown_contract_or_capability_is_refused() -> None: + unknown = Manifest("server", (), (Required("clock", Version(1, 0), ("read",)),), (), ()) + assert codes(problems(provider(), unknown)) == ["ASSEMBLY_CONTRACT_UNKNOWN"] + assert codes(problems(provider(capabilities=("read", "write", "purge")))) == [ + "ASSEMBLY_CAPABILITY_UNKNOWN" + ] + assert codes(problems(provider(), consumer(capabilities=("purge",)))) == [ + "ASSEMBLY_CAPABILITY_UNKNOWN" + ] + + +def test_every_problem_is_reported_together() -> None: + with pytest.raises(AssemblyRefused) as raised: + check_assembly( + (provider(capabilities=("read",)), consumer("1.2")), + {"cache": CACHE}, + {("alpha", "cache"): WithoutPut()}, + ) + assert raised.value.code == "ASSEMBLY_REFUSED" + assert codes(raised.value.problems) == [ + "ASSEMBLY_VERSION_INCOMPATIBLE", + "ASSEMBLY_CAPABILITY_MISSING", + ] + assert str(raised.value).startswith("ASSEMBLY_REFUSED: ") + + +def api_module(namespace: str, module: str, **attributes: object) -> Callable[[], None]: + name = f"{namespace}.{module}.api" + sys.modules[name] = types.ModuleType(name) + for key, value in attributes.items(): + setattr(sys.modules[name], key, value) + + def cleanup() -> None: + sys.modules.pop(name, None) + + return cleanup + + +def test_implementations_are_loaded_from_the_module_s_api_package_only() -> None: + namespace = load_identity().python_namespace + cleanup = api_module(namespace, "alpha", Cache=Whole) + try: + loaded = load_implementations(namespace, (provider(),)) + assert loaded == {("alpha", "cache"): Whole} + finally: + cleanup() + + +def test_an_implementation_that_is_not_there_is_refused() -> None: + namespace = load_identity().python_namespace + with pytest.raises(AssemblyRefused) as raised: + load_implementations(namespace, (provider(module="absent"),)) + assert codes(raised.value.problems) == ["ASSEMBLY_IMPLEMENTATION_MISSING"] + cleanup = api_module(namespace, "alpha") + try: + with pytest.raises(AssemblyRefused) as raised: + load_implementations(namespace, (provider(),)) + assert codes(raised.value.problems) == ["ASSEMBLY_IMPLEMENTATION_MISSING"] + finally: + cleanup() + + +def test_the_installed_modules_assemble() -> None: + identity = load_identity() + modules = [manifest.module for manifest in installed_manifests(identity)] + assert "contracts" in modules + check_installed_assembly(identity) + + +def test_the_server_does_not_start_when_assembly_is_refused( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + def refuse(identity: object) -> None: + raise AssemblyRefused(["ASSEMBLY_CAPABILITY_MISSING: planted"]) + + monkeypatch.setattr(cli, "check_installed_assembly", refuse) + assert cli.main([]) == 3 + assert "ASSEMBLY_CAPABILITY_MISSING: planted" in capsys.readouterr().err diff --git a/packages/contracts/tests/test_http_api_v1.py b/tests/contracts/test_http_api_v1.py similarity index 97% rename from packages/contracts/tests/test_http_api_v1.py rename to tests/contracts/test_http_api_v1.py index cd58055f..ef765e78 100644 --- a/packages/contracts/tests/test_http_api_v1.py +++ b/tests/contracts/test_http_api_v1.py @@ -7,7 +7,7 @@ from orvex.contracts.http_api import v1 from pydantic import BaseModel, ValidationError -EXAMPLES = Path(__file__).resolve().parents[3] / "contracts" / "http-api" / "v1" / "examples" +EXAMPLES = Path(__file__).resolve().parents[2] / "contracts" / "http-api" / "v1" / "examples" def model_for(example: Path) -> type[BaseModel]: diff --git a/tests/tools/test_consumer_boundaries.py b/tests/tools/test_consumer_boundaries.py new file mode 100644 index 00000000..ddfb7183 --- /dev/null +++ b/tests/tools/test_consumer_boundaries.py @@ -0,0 +1,115 @@ +"""Consumer boundaries (#486 G2), in report mode until step 7: the planted imports are caught.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from tools.checks.boundaries import run +from tools.checks.consumers import consumer_findings, consumer_scan, consumer_summary +from tools.repository import REPOSITORY_ROOT, identity + +from .test_checks import make_repository + +NAMESPACE = str(identity(REPOSITORY_ROOT)["python_namespace"]) +ASSEMBLY = "packages/server/src/orvex/server/assembly.py" + +PLANTED = { + "server": ("packages/server/src/orvex/server/x.py", "from orvex.store.lake import open_lake\n"), + "server test": ("packages/server/tests/test_x.py", "import orvex.access.errors\n"), + "runner": ("conformance/runners/x.py", "from orvex.semantic import errors\n"), + "tool": ("tools/x.py", "from orvex.transform.incremental import plan\n"), + "repository test": ("tests/x/test_y.py", "import orvex.semantic.evidence as e\n"), + "dynamic": ( + "tests/x/test_z.py", + 'import importlib\nimportlib.import_module("orvex.store.lake")\n', + ), + "dunder": ("tools/y.py", '__import__("orvex.mcp.server")\n'), + "api outside assembly": ("packages/server/src/orvex/server/y.py", "import orvex.store.api\n"), + "internal in assembly": (ASSEMBLY, "from orvex.store.lake import x\n"), +} + + +def findings(relative: str, source: str) -> list[str]: + return [f.code for f in consumer_findings(ast.parse(source), NAMESPACE, relative)] + + +@pytest.mark.parametrize(("relative", "source"), PLANTED.values(), ids=PLANTED.keys()) +def test_a_planted_internal_import_is_caught(relative: str, source: str) -> None: + assert findings(relative, source) == ["CONSUMER_BOUNDARY_VIOLATION"] + + +ALLOWED = { + "contracts": ("tools/x.py", "from orvex.contracts.store.v1 import StoreError\n"), + "contracts package": ( + "tests/test_x.py", + f"from {NAMESPACE} import contracts\nimport {NAMESPACE}\n", + ), + "the server's own modules": ( + "packages/server/src/orvex/server/x.py", + "import orvex.server.cli\n", + ), + "server tests of the server": ("packages/server/tests/t.py", "from orvex.server import cli\n"), + "api in assembly": (ASSEMBLY, "import orvex.store.api\nfrom orvex.semantic.api import X\n"), + "relative": ("packages/server/src/orvex/server/x.py", "from . import cli\n"), + "third party": ("tools/x.py", "import duckdb\nfrom tools.repository import identity\n"), + "computed dynamic import": ("tools/x.py", "import importlib\nimportlib.import_module(name)\n"), +} + + +@pytest.mark.parametrize(("relative", "source"), ALLOWED.values(), ids=ALLOWED.keys()) +def test_contracts_own_code_and_api_in_assembly_are_allowed(relative: str, source: str) -> None: + assert findings(relative, source) == [] + + +def test_a_tests_import_of_the_server_from_outside_it_is_caught() -> None: + assert findings("tests/x.py", "from orvex.server.cli import main\n") == [ + "CONSUMER_BOUNDARY_VIOLATION" + ] + + +def test_the_scan_covers_the_consumers_and_leaves_the_modules_to_the_module_rule( + tmp_path: Path, +) -> None: + files = dict(PLANTED.values()) + files["packages/store/src/orvex/store/x.py"] = "import orvex.store.lake\n" + files["conformance/semantic-corpus/x.py"] = "import orvex.store.lake\n" + found = consumer_scan(make_repository(tmp_path, files)) + assert sorted({f.path for f in found}) == sorted({relative for relative, _ in PLANTED.values()}) + summary = consumer_summary(found) + assert summary.startswith(f"consumer boundaries (report mode until #486 step 7): {len(found)}") + assert "server 4" in summary + assert "runners 1" in summary + assert "tools 2" in summary + assert "tests 2" in summary + + +def test_report_mode_prints_every_violation_and_does_not_fail( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + empty = tmp_path / "empty.toml" + empty.write_text("") + (tmp_path / "r").mkdir() + repository = make_repository(tmp_path / "r", dict(PLANTED.values())) + assert run(repository, empty, empty) == 0 + out = capsys.readouterr().out + for relative, _ in PLANTED.values(): + assert f"CONSUMER_BOUNDARY_VIOLATION: {relative}:" in out + assert "consumer boundaries (report mode until #486 step 7)" in out + + +def test_a_module_boundary_violation_still_fails(tmp_path: Path) -> None: + empty = tmp_path / "empty.toml" + empty.write_text("") + files = {"packages/store/src/orvex/store/x.py": "import orvex.access.errors\n"} + (tmp_path / "r").mkdir() + assert run(make_repository(tmp_path / "r", files), empty, empty) == 1 + + +def test_the_repository_report_lists_what_remains() -> None: + found = consumer_scan(REPOSITORY_ROOT) + areas = consumer_summary(found) + assert all(f.code == "CONSUMER_BOUNDARY_VIOLATION" for f in found) + assert "server" in areas diff --git a/tests/tools/test_interface_compat.py b/tests/tools/test_interface_compat.py new file mode 100644 index 00000000..45600e9d --- /dev/null +++ b/tests/tools/test_interface_compat.py @@ -0,0 +1,298 @@ +"""Interface-contract compatibility with the last release (#486 G3), and the record it compares.""" + +from __future__ import annotations + +import copy +import json +import subprocess +from pathlib import Path +from typing import Any, Protocol + +import orvex.contracts +import pytest +from orvex.contracts.binding import Bound +from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.kit import Kit, KitCase +from orvex.contracts.registry import interface_contracts, kits + +from tools.contracts import interface_compat, interface_record +from tools.contracts.interface_compat import incompatibilities, last_release, released_record +from tools.contracts.interface_record import RECORD, build_record, render +from tools.repository import REPOSITORY_ROOT + + +class MeterError(Exception): + def __init__(self, code: str, message: str, details: dict[str, str]) -> None: + super().__init__(f"{code}: {message}") + + +class Meter(Protocol): + def read(self, meter: str) -> int: ... + + def reset(self, meter: str) -> None: ... + + +def check(meter: Bound) -> None: + meter.read("m") + + +def contract(**changes: Any) -> InterfaceContract: # noqa: ANN401 + fields: dict[str, Any] = { + "name": "meter", + "version": Version(1, 0), + "protocol": Meter, + "error_type": MeterError, + "capabilities": ("read", "write"), + "errors": ("METER_UNKNOWN", "METER_LOCKED"), + "operations": ( + Operation("read", "read", ("METER_UNKNOWN",)), + Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED")), + ), + } + fields.update(changes) + return InterfaceContract(**fields) + + +def kit(of: InterfaceContract, *cases: KitCase) -> Kit: + chosen = cases or ( + KitCase("read.known", "read", "a known meter reads its count", check), + KitCase("reset.zero", "reset", "a reset meter reads zero", check), + ) + return Kit(of, recorded=len(chosen), cases=chosen) + + +def recorded(of: InterfaceContract, the_kit: Kit | None = None) -> dict[str, Any]: + return build_record({of.name: of}, {of.name: the_kit or kit(of)}) + + +RELEASED = recorded(contract()) + + +def codes(found: list[str]) -> list[str]: + return [problem.split(":", 1)[0] for problem in found] + + +class ReadOnly(Protocol): + def read(self, meter: str) -> int: ... + + +class Renamed(Protocol): + def read(self, name: str) -> int: ... + + def reset(self, meter: str) -> None: ... + + +class Retyped(Protocol): + def read(self, meter: str) -> str: ... + + def reset(self, meter: str) -> None: ... + + +class MoreRequired(Protocol): + def read(self, meter: str, at: int) -> int: ... + + def reset(self, meter: str) -> None: ... + + +class MoreOptional(Protocol): + def read(self, meter: str, at: int | None = None) -> int: ... + + def reset(self, meter: str) -> None: ... + + def total(self) -> int: ... + + +def removed_operation() -> dict[str, Any]: + reduced = contract( + protocol=ReadOnly, + capabilities=("read",), + operations=(Operation("read", "read", ("METER_UNKNOWN",)),), + ) + return recorded( + reduced, kit(reduced, KitCase("read.known", "read", "a known meter reads its count", check)) + ) + + +def renamed_error() -> dict[str, Any]: + return recorded( + contract( + errors=("METER_NOT_FOUND", "METER_LOCKED"), + operations=( + Operation("read", "read", ("METER_NOT_FOUND",)), + Operation("reset", "write", ("METER_NOT_FOUND", "METER_LOCKED")), + ), + ) + ) + + +def changed_expectation() -> dict[str, Any]: + of = contract() + return recorded( + of, + kit( + of, + KitCase("read.known", "read", "a known meter reads its count, or zero", check), + KitCase("reset.zero", "reset", "a reset meter reads zero", check), + ), + ) + + +class TestPlantedViolations: + def test_a_removed_operation_fails(self) -> None: + found = codes(incompatibilities(RELEASED, removed_operation())) + assert "INTERFACE_OPERATION_REMOVED" in found + assert "INTERFACE_CAPABILITY_REMOVED" in found + assert "INTERFACE_KIT_CASE_REMOVED" in found + + def test_a_renamed_error_code_fails(self) -> None: + assert codes(incompatibilities(RELEASED, renamed_error())) == [ + "INTERFACE_ERROR_CODE_REMOVED" + ] + + def test_a_changed_kit_expectation_fails(self) -> None: + assert codes(incompatibilities(RELEASED, changed_expectation())) == [ + "INTERFACE_KIT_EXPECTATION_CHANGED" + ] + + @pytest.mark.parametrize("protocol", [Renamed, Retyped, MoreRequired], ids=lambda p: p.__name__) + def test_a_changed_signature_fails(self, protocol: type) -> None: + assert codes(incompatibilities(RELEASED, recorded(contract(protocol=protocol)))) == [ + "INTERFACE_SIGNATURE_CHANGED" + ] + + def test_a_moved_operation_fails(self) -> None: + moved = contract( + operations=( + Operation("read", "write", ("METER_UNKNOWN",)), + Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED")), + ), + capabilities=("write",), + ) + found = codes(incompatibilities(RELEASED, recorded(moved))) + assert "INTERFACE_SIGNATURE_CHANGED" in found + + def test_a_removed_contract_and_a_lowered_version_fail(self) -> None: + assert codes(incompatibilities(RELEASED, {"contracts": {}})) == [ + "INTERFACE_CONTRACT_REMOVED" + ] + newer = recorded(contract(version=Version(1, 3))) + assert codes(incompatibilities(newer, RELEASED)) == ["INTERFACE_VERSION_LOWERED"] + + +class TestCompatibleChanges: + def test_the_same_contract_is_compatible(self) -> None: + assert incompatibilities(RELEASED, copy.deepcopy(RELEASED)) == [] + + def test_appending_is_compatible(self) -> None: + grown = contract( + version=Version(1, 1), + protocol=MoreOptional, + capabilities=("read", "write", "sum"), + errors=("METER_UNKNOWN", "METER_LOCKED", "METER_OVERFLOW"), + operations=( + Operation("read", "read", ("METER_UNKNOWN",)), + Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED")), + Operation("total", "sum", ("METER_OVERFLOW",)), + ), + ) + cases = ( + KitCase("read.known", "read", "a known meter reads its count", check), + KitCase("reset.zero", "reset", "a reset meter reads zero", check), + KitCase("total.sums", "total", "the total sums every meter", check), + ) + assert incompatibilities(RELEASED, recorded(grown, kit(grown, *cases))) == [] + assert incompatibilities({"contracts": {}}, RELEASED) == [] + + @pytest.mark.parametrize("build", [removed_operation, renamed_error, changed_expectation]) + def test_a_new_major_version_may_break(self, build: Any) -> None: # noqa: ANN401 + broken = build() + broken["contracts"]["meter"]["version"] = "2.0" + assert incompatibilities(RELEASED, broken) == [] + + +class TestRecord: + def test_the_record_holds_each_contract_with_its_kit(self) -> None: + entry = RELEASED["contracts"]["meter"] + assert entry["version"] == "1.0" + assert sorted(entry["operations"]) == ["read", "reset"] + assert entry["kit"]["recorded"] == 2 + + def test_a_contract_without_a_kit_and_a_kit_without_a_contract_are_refused(self) -> None: + with pytest.raises(SystemExit, match="INTERFACE_KIT_MISSING"): + build_record({"meter": contract()}, {}) + with pytest.raises(SystemExit, match="INTERFACE_KIT_ORPHANED"): + build_record({}, {"meter": kit(contract())}) + + def test_the_committed_record_is_current(self) -> None: + current = build_record(interface_contracts(orvex.contracts), kits(orvex.contracts)) + assert RECORD.read_text("utf-8") == render(current) + assert interface_record.main(["--check"]) == 0 + + def test_a_stale_record_fails_the_check( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + stale = tmp_path / "record.json" + stale.write_text(render(RELEASED)) + monkeypatch.setattr(interface_record, "RECORD", stale) + assert interface_record.main(["--check"]) == 1 + assert interface_record.main([]) == 0 + assert stale.read_text("utf-8") == RECORD.read_text("utf-8") + + +def git(root: Path, *arguments: str) -> None: + subprocess.run(["git", *arguments], cwd=root, check=True, capture_output=True) + + +class TestRelease: + def write(self, root: Path, record: dict[str, Any] | None) -> None: + path = root / RECORD.relative_to(REPOSITORY_ROOT) + path.parent.mkdir(parents=True, exist_ok=True) + if record is not None: + path.write_text(render(record)) + (root / "README.md").write_text(json.dumps(record)) + + def repository(self, tmp_path: Path, released: dict[str, Any] | None) -> Path: + git(tmp_path, "init", "-q", "-b", "main") + git(tmp_path, "config", "user.email", "t@example.invalid") + git(tmp_path, "config", "user.name", "t") + self.write(tmp_path, released) + git(tmp_path, "add", "-A") + git(tmp_path, "commit", "-qm", "release") + git(tmp_path, "tag", "-a", "v0.4.0-alpha.1", "-m", "release") + git(tmp_path, "tag", "-a", "store-v0.4.0", "-m", "module tag") + git(tmp_path, "tag", "v0.10.0a1") + self.write(tmp_path, {"contracts": {}}) + git(tmp_path, "add", "-A") + git(tmp_path, "commit", "-qm", "later") + return tmp_path + + def test_the_record_of_the_last_product_release_is_read(self, tmp_path: Path) -> None: + root = self.repository(tmp_path, RELEASED) + assert last_release(root) == "v0.4.0-alpha.1" + assert released_record(root, "v0.4.0-alpha.1") == RELEASED + + def test_a_release_without_a_record_released_no_interface_contract( + self, tmp_path: Path + ) -> None: + root = self.repository(tmp_path, None) + assert released_record(root, "v0.4.0-alpha.1") is None + + def test_no_release_tag_fails_closed(self, tmp_path: Path) -> None: + git(tmp_path, "init", "-q", "-b", "main") + git(tmp_path, "config", "user.email", "t@example.invalid") + git(tmp_path, "config", "user.name", "t") + (tmp_path / "a").write_text("a") + git(tmp_path, "add", "-A") + git(tmp_path, "commit", "-qm", "first") + with pytest.raises(SystemExit, match="INTERFACE_RELEASE_UNKNOWN"): + last_release(tmp_path) + + def test_the_check_fails_on_a_break_since_the_release( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] + ) -> None: + root = self.repository(tmp_path, RELEASED) + monkeypatch.setattr(interface_compat, "current_record", renamed_error) + assert interface_compat.run(root) == 1 + assert "INTERFACE_ERROR_CODE_REMOVED" in capsys.readouterr().out + monkeypatch.setattr(interface_compat, "current_record", lambda: RELEASED) + assert interface_compat.run(root) == 0 diff --git a/tests/tools/test_standalone.py b/tests/tools/test_standalone.py new file mode 100644 index 00000000..2c9f2b01 --- /dev/null +++ b/tests/tools/test_standalone.py @@ -0,0 +1,130 @@ +"""The standalone job's checks (#486 G4): declared dependencies equal the used ones, and a +module's tests import no repository helper. The planted violations each fail.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tools.checks.module_dependencies import ( + dependency_problems, + imported_modules, + suite_problems, +) +from tools.ci.standalone import constraints, outside_repository +from tools.repository import REPOSITORY_ROOT + +OWNERS = { + "orvex.store": "orvex-store", + "orvex.store.lake": "orvex-store", + "orvex.contracts": "orvex-contracts", + "orvex.contracts.store.v1": "orvex-contracts", + "duckdb": "duckdb", + "pyarrow": "pyarrow", + "pyarrow.compute": "pyarrow", + "yaml": "PyYAML", + "pytest": "pytest", + "tests.support": None, +} + + +def owner_of(name: str) -> str | None: + return OWNERS.get(name) + + +def codes(found: list[str]) -> list[str]: + return sorted(problem.split(":", 1)[0] for problem in found) + + +SOURCE = "import json\nimport duckdb\nfrom orvex.contracts.store.v1 import X\nfrom . import y\n" + + +def test_imports_are_read_with_their_full_names() -> None: + assert imported_modules(SOURCE + "import pyarrow.compute as pc\n") == { + "json", + "duckdb", + "orvex.contracts.store.v1", + "pyarrow.compute", + } + + +def test_declared_dependencies_equal_to_the_used_ones_pass() -> None: + imports = imported_modules(SOURCE) | {"orvex.store.lake"} + declared = ["duckdb", "orvex-contracts"] + assert dependency_problems("orvex-store", imports, owner_of, declared, declared) == [] + + +def test_a_planted_undeclared_dependency_fails() -> None: + imports = imported_modules(SOURCE) | {"pyarrow.compute"} + declared = ["duckdb", "orvex-contracts"] + found = dependency_problems("orvex-store", imports, owner_of, declared, declared) + assert codes(found) == ["MODULE_DEPENDENCY_UNDECLARED"] + assert "pyarrow" in found[0] + + +def test_an_import_nothing_installed_provides_fails() -> None: + found = dependency_problems("orvex-store", {"missing_package"}, owner_of, [], []) + assert codes(found) == ["MODULE_DEPENDENCY_UNDECLARED"] + + +def test_a_declared_dependency_not_used_fails() -> None: + declared = ["duckdb", "orvex-contracts", "pyarrow"] + found = dependency_problems( + "orvex-store", imported_modules(SOURCE), owner_of, declared, declared + ) + assert codes(found) == ["MODULE_DEPENDENCY_UNUSED"] + + +def test_a_manifest_that_disagrees_with_the_distribution_fails() -> None: + declared = ["duckdb", "orvex-contracts"] + found = dependency_problems( + "orvex-store", + imported_modules(SOURCE), + owner_of, + declared, + ["duckdb", "orvex-contracts", "PyArrow"], + ) + assert codes(found) == ["MODULE_MANIFEST_DEPENDENCIES_DIFFER"] + + +@pytest.mark.parametrize( + "source", + [ + "from tests.support.access_directory import World\n", + "import conformance.runners.semantic_corpus\n", + "from tools.repository import REPOSITORY_ROOT\n", + "import tests\n", + ], +) +def test_a_planted_test_import_of_a_repository_helper_fails(source: str) -> None: + found = suite_problems( + "orvex-store", {"test_a.py": imported_modules(source)}, owner_of, ["duckdb"], [] + ) + assert codes(found) == ["MODULE_TEST_IMPORTS_REPOSITORY"] + assert "test_a.py" in found[0] + + +def test_test_dependencies_must_be_declared_and_used() -> None: + imports = {"t.py": {"pytest", "yaml", "duckdb", "orvex.store.lake"}} + assert suite_problems("orvex-store", imports, owner_of, ["duckdb"], ["pytest", "pyyaml"]) == [] + assert codes(suite_problems("orvex-store", imports, owner_of, ["duckdb"], ["pytest"])) == [ + "MODULE_TEST_DEPENDENCY_UNDECLARED" + ] + assert codes( + suite_problems( + "orvex-store", imports, owner_of, ["duckdb"], ["pytest", "pyyaml", "hypothesis"] + ) + ) == ["MODULE_TEST_DEPENDENCY_UNUSED"] + + +def test_the_job_works_outside_the_repository(tmp_path: Path) -> None: + assert outside_repository(tmp_path) == tmp_path + with pytest.raises(SystemExit, match="STANDALONE_INSIDE_REPOSITORY"): + outside_repository(REPOSITORY_ROOT / "build") + + +def test_the_environment_is_constrained_to_the_lock() -> None: + pinned = constraints(REPOSITORY_ROOT / "pylock.toml") + assert "pydantic==2.14.3" in pinned + assert all("==" in line for line in pinned) From 06ed99a0e07a103dbcd9fda0c5a259a472ad879d Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sat, 14 Nov 2026 21:34:49 -0400 Subject: [PATCH 02/21] feat(contracts): interface contracts, kits, manifests and their checks Step 1 of #486 (foundations): the interface-contract format, conformance kits with a recorded count and no skips, the in-process and serializing bindings, the module manifest, the assembly capability check at server start, the consumer boundaries in report mode, the compatibility check against the last release, the standalone job on orvex-contracts, and ADR-0033 (Proposed). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .github/workflows/ci-alerts.yml | 1 + .github/workflows/interfaces.yml | 56 +++++ .github/workflows/scripts/ci_alerts.toml | 7 + ...-interface-contracts-kits-and-manifests.md | 22 ++ docs/adr/README.md | 3 +- docs/process/git-workflow.md | 10 + packages/contracts/README.md | 7 +- packages/contracts/interface-record.json | 3 + packages/contracts/pyproject.toml | 4 +- .../contracts/src/orvex/contracts/__init__.py | 7 +- .../contracts/src/orvex/contracts/binding.py | 80 ++++++ .../contracts/src/orvex/contracts/codec.py | 160 ++++++++++++ .../src/orvex/contracts/interface.py | 234 ++++++++++++++++++ .../src/orvex/contracts/interface_errors.py | 22 ++ packages/contracts/src/orvex/contracts/kit.py | 73 ++++++ .../contracts/src/orvex/contracts/kit_run.py | 83 +++++++ .../contracts/src/orvex/contracts/manifest.py | 168 +++++++++++++ .../contracts/src/orvex/contracts/module.toml | 7 + .../contracts/src/orvex/contracts/registry.py | 48 ++++ .../src/orvex/contracts/serializable.py | 105 ++++++++ packages/server/src/orvex/server/assembly.py | 163 ++++++++++++ packages/server/src/orvex/server/cli.py | 7 + tools/checks/boundaries.py | 7 + tools/checks/consumers.py | 112 +++++++++ tools/checks/module_dependencies.py | 196 +++++++++++++++ tools/ci/checks.toml | 33 +++ tools/ci/standalone.py | 155 ++++++++++++ tools/contracts/interface_compat.py | 162 ++++++++++++ tools/contracts/interface_record.py | 75 ++++++ 29 files changed, 2005 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/interfaces.yml create mode 100644 docs/adr/0033-interface-contracts-kits-and-manifests.md create mode 100644 packages/contracts/interface-record.json create mode 100644 packages/contracts/src/orvex/contracts/binding.py create mode 100644 packages/contracts/src/orvex/contracts/codec.py create mode 100644 packages/contracts/src/orvex/contracts/interface.py create mode 100644 packages/contracts/src/orvex/contracts/interface_errors.py create mode 100644 packages/contracts/src/orvex/contracts/kit.py create mode 100644 packages/contracts/src/orvex/contracts/kit_run.py create mode 100644 packages/contracts/src/orvex/contracts/manifest.py create mode 100644 packages/contracts/src/orvex/contracts/module.toml create mode 100644 packages/contracts/src/orvex/contracts/registry.py create mode 100644 packages/contracts/src/orvex/contracts/serializable.py create mode 100644 packages/server/src/orvex/server/assembly.py create mode 100644 tools/checks/consumers.py create mode 100644 tools/checks/module_dependencies.py create mode 100644 tools/ci/standalone.py create mode 100644 tools/contracts/interface_compat.py create mode 100644 tools/contracts/interface_record.py diff --git a/.github/workflows/ci-alerts.yml b/.github/workflows/ci-alerts.yml index e31ad89f..df732025 100644 --- a/.github/workflows/ci-alerts.yml +++ b/.github/workflows/ci-alerts.yml @@ -25,6 +25,7 @@ on: - property-nightly - ossie-upstream - ci-alerts-selftest + - interfaces types: [completed] schedule: - cron: "41 */6 * * *" diff --git a/.github/workflows/interfaces.yml b/.github/workflows/interfaces.yml new file mode 100644 index 00000000..373fbe4d --- /dev/null +++ b/.github/workflows/interfaces.yml @@ -0,0 +1,56 @@ +# Interface contracts (#486): their record is current and compatible with the last release +# (G3), and a module installs and passes its own tests alone, outside the repository (G4). +name: interfaces + +on: + pull_request: + push: + branches: [main] + +# A newer commit on a pull request cancels the pull request's older runs (#386). Pushes to one +# branch (main) share a group: a running one is never cancelled, and only the newest waiting +# one is kept, since it tests everything the older ones would (#391). +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + interface-contracts: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The whole history and its tags: the check reads the record of the last release. + fetch-depth: 0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.15" + - name: Install from the lock file and the contracts package + run: | + python -m pip install --quiet pip==26.3 + python -m pip install --quiet -r pylock.toml + python -m pip install --quiet --no-deps --no-build-isolation -e packages/contracts + - name: The interface record is current + run: python -m tools.contracts.interface_record --check + - name: Compatible with the last release + run: python -m tools.contracts.interface_compat + + standalone: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.15" + - name: Install the build tools from the lock file + run: | + python -m pip install --quiet pip==26.3 + python -m pip install --quiet -r pylock.toml + - name: The contracts module alone, outside the repository + run: python -m tools.ci.standalone contracts diff --git a/.github/workflows/scripts/ci_alerts.toml b/.github/workflows/scripts/ci_alerts.toml index cd102aa4..bef82702 100644 --- a/.github/workflows/scripts/ci_alerts.toml +++ b/.github/workflows/scripts/ci_alerts.toml @@ -96,3 +96,10 @@ workflow = "ci-alerts-selftest.yml" events = ["workflow_dispatch"] required_jobs = ["selftest"] cadence = "none" + +[[watch]] +# Interface contracts (#486): the record and its compatibility, and the standalone module. +workflow = "interfaces.yml" +events = ["push"] +required_jobs = ["interface-contracts", "standalone"] +cadence = "push" diff --git a/docs/adr/0033-interface-contracts-kits-and-manifests.md b/docs/adr/0033-interface-contracts-kits-and-manifests.md new file mode 100644 index 00000000..86b55658 --- /dev/null +++ b/docs/adr/0033-interface-contracts-kits-and-manifests.md @@ -0,0 +1,22 @@ +# 0033 — Interface contracts, conformance kits and module manifests (amends 0004 rules 2–4) + +**Status:** Proposed (2026-11-14, #486) + +## Context +ADR-0004 makes modules talk only through contracts, but its contracts are data contracts: they say what crosses a boundary, never which operations a module offers. On 2026-11-14 no module imported another module's internals, yet `server`, the conformance runners, `tools/` and the repository tests used 65 internal paths of the modules (#486 §1), the boundaries check exempted `server` as the composition root, and no module had ever been installed or tested outside the repository. Nothing proved that another implementation of a module could replace it. + +## Decision +1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version, groups the operations into capabilities and lists the error codes each may raise through the contract's error type. Only values cross: the annotated inputs and outputs are plain values, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`; its kit in `orvex.contracts..kit` (`orvex.contracts.registry`). +2. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise and their expectation in words, a recorded case count and no skip list. A kit leaves no operation without a case. It runs on a fresh implementation per case, once in process and once through the serializing binding (`orvex.contracts.kit_run`, `binding`, `codec`), and passes only when every case passes both ways with the same results; a skip is a failure. An implementation that passes its contract's whole kit may replace the module. +3. **Consumers know contracts, not modules** (amends rule 3). `server` stays the only composition root, but only `server/assembly.py` names implementations, and only through each module's public `api` package. Every other consumer (`server`, the conformance runners, `tools/`, the repository tests) imports only `orvex.contracts`. `tools/checks/consumers.py` enforces it, in report mode until #486 step 7 makes it blocking and removes the `server` exemption. +4. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` attribute that implements each) and requires (version, capabilities), and the distributions its code and its tests import. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation or more than one, a version that does not provide the required one, a missing capability, or an implementation lacking an operation of what it claims. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. +5. **Compatibility of behavior.** The record of every interface contract and its kit (`packages/contracts/interface-record.json`, `tools/contracts/interface_record.py`) is compared with the record at the last product release (`tools/contracts/interface_compat.py`): within a major version operations, signatures, capabilities, error codes and kit cases with their expectations may only be added. A break needs a new major version (rule 6 of ADR-0004 applies to it). +6. **Standalone modules.** `tools/ci/standalone.py` builds a module's sdist and, from it, its wheel, installs it in a clean environment outside the repository with only `orvex-contracts` beside it, checks that its imports equal its declared dependencies and that its tests import no repository helper, and runs its own tests there, where a skipped test fails. + +## Alternatives considered +- **Interface contracts as language-neutral documents with generated protocols:** symmetrical with the data contracts, but it adds a generator and a second description of each signature to keep in step; the record of rule 5 already gives a language-neutral view of each interface for comparison. +- **A second, complete implementation of every module** (#486 revision 3): proves substitution but duplicates about 41,000 lines; the kits, the assembly check and one real substitution (the semantic result cache, #486 step 5) prove it at a fraction of the cost. +- **Process isolation now:** belongs to the workers of #156; rule 1's value-only crossing and the serializing binding make the contracts ready for it. + +## Consequences +Every module gains a contract, a kit, a manifest and an `api` package, one module per step of #486, and its consumers move onto them; module error classes used across modules move into `orvex.contracts`. CI gains the interface record and compatibility check and the standalone job. The documents that say `server` knows the modules (`architecture/overview.md`, `AGENTS.md` §5, `engineering.md` §2 and §8, `modules/contracts.md`) are corrected when this ADR is accepted. diff --git a/docs/adr/README.md b/docs/adr/README.md index b0c2ac7f..e84a1a10 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -9,7 +9,7 @@ Records 0001–0026 were approved by the Owner during the design session of 2026 | [0001](0001-record-architecture-decisions.md) | Record architecture decisions | Accepted | | [0002](0002-dependency-selection-policy.md) | Open standards first; dependency selection policy | Accepted | | [0003](0003-python-backend-and-svelte-web.md) | Python backend with FastAPI; SvelteKit web | Accepted | -| [0004](0004-modules-contracts-and-independent-versions.md) | Modules with contracts, own state and independent versions | Accepted | +| [0004](0004-modules-contracts-and-independent-versions.md) | Modules with contracts, own state and independent versions | Accepted; rules 2–4 amended by 0033 once accepted | | [0005](0005-local-and-server-runtime-modes.md) | Local zero-configuration mode and opt-in server mode | Accepted; amended in part by 0027 | | [0006](0006-duckdb-and-ducklake.md) | DuckDB as the only analytical engine; DuckLake as the warehouse | Accepted; superseded in part by 0027 and 0028 | | [0007](0007-projects-as-files.md) | Projects as files in Git; module state separate | Accepted | @@ -38,3 +38,4 @@ Records 0001–0026 were approved by the Owner during the design session of 2026 | [0030](0030-ossie-version-evolution.md) | Evolving with Apache Ossie: version registry, staged reading, three identities and the Ossie-first policy (amends 0009 and 0012) | Accepted | | [0031](0031-source-isolation-boundary.md) | Source isolation boundary: every external database read in a worker without the lake (makes 0025 precise) | Accepted | | [0032](0032-security-cases-follow-adr-0027.md) | Security cases follow ADR-0027 §3 for views, macros and overlapping locations (ratifies 0027) | Proposed | +| [0033](0033-interface-contracts-kits-and-manifests.md) | Interface contracts, conformance kits and module manifests (amends 0004 rules 2–4) | Proposed | diff --git a/docs/process/git-workflow.md b/docs/process/git-workflow.md index 3774641e..aaca7196 100644 --- a/docs/process/git-workflow.md +++ b/docs/process/git-workflow.md @@ -114,6 +114,16 @@ packages/contracts/src/orvex/contracts/ generated contract types !packages/contracts/src/orvex/contracts/__init__.py hand-written: counted !packages/contracts/src/orvex/contracts/contracts.toml hand-written: counted !packages/contracts/src/orvex/contracts/py.typed hand-written: counted +!packages/contracts/src/orvex/contracts/binding.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/codec.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/interface.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/interface_errors.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/kit.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/kit_run.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/manifest.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/module.toml hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/registry.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/serializable.py hand-written: counted (#486, interface contracts) tests/contracts/generated/ generated contract fixtures !tests/contracts/generated/__init__.py hand-written: counted web/src/lib/contracts/ generated TypeScript types diff --git a/packages/contracts/README.md b/packages/contracts/README.md index b3dc7d21..21bce09c 100644 --- a/packages/contracts/README.md +++ b/packages/contracts/README.md @@ -1,6 +1,10 @@ # `orvex-contracts` — `orvex.contracts` -Python types generated from [`contracts/`](../../contracts/README.md) by `python -m tools.contracts.generate`. Every module may import this package; nothing here is written by hand. +Every module may import this package. It holds: + +- Python types generated from [`contracts/`](../../contracts/README.md) by `python -m tools.contracts.generate` (`/v.py`), never edited by hand; +- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, version), `kit.py` and `kit_run.py` (conformance kits: cases, recorded count, no skips, run in process and serialized), `binding.py`, `codec.py` and `serializable.py` (what may cross a contract), `manifest.py` (`module.toml`), `registry.py` and `interface_errors.py`; +- `module.toml`, this module's manifest, and `interface-record.json` (beside `pyproject.toml`), the record of every interface contract that `python -m tools.contracts.interface_record` writes and `python -m tools.contracts.interface_compat` compares with the last release. Module pack: [`docs/modules/contracts.md`](../../docs/modules/contracts.md). @@ -8,4 +12,5 @@ Module pack: [`docs/modules/contracts.md`](../../docs/modules/contracts.md). ```bash pytest packages/contracts +python -m tools.ci.standalone contracts # built, installed and tested alone, outside the repository ``` diff --git a/packages/contracts/interface-record.json b/packages/contracts/interface-record.json new file mode 100644 index 00000000..50cf3182 --- /dev/null +++ b/packages/contracts/interface-record.json @@ -0,0 +1,3 @@ +{ + "contracts": {} +} diff --git a/packages/contracts/pyproject.toml b/packages/contracts/pyproject.toml index e0ceddf3..67c55300 100644 --- a/packages/contracts/pyproject.toml +++ b/packages/contracts/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "orvex-contracts" version = "0.3.0" -description = "Types generated from the language-neutral contracts in contracts/. Never edited by hand." +description = "The contracts every module may import: generated contract types and the interface-contract format." readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE"] @@ -17,4 +17,4 @@ where = ["src"] namespaces = true [tool.setuptools.package-data] -"orvex.contracts" = ["py.typed", "contracts.toml", "*/v*_data/*.json", "*/v*_data/*/*.json"] +"orvex.contracts" = ["py.typed", "contracts.toml", "module.toml", "*/v*_data/*.json", "*/v*_data/*/*.json"] diff --git a/packages/contracts/src/orvex/contracts/__init__.py b/packages/contracts/src/orvex/contracts/__init__.py index a4116352..77725daf 100644 --- a/packages/contracts/src/orvex/contracts/__init__.py +++ b/packages/contracts/src/orvex/contracts/__init__.py @@ -1,4 +1,9 @@ -"""Types generated from the language-neutral contracts in `contracts/`. Never edited by hand.""" +"""The contracts every module may import: types generated from `contracts/`, and the format of +interface contracts, their kits and module manifests (ADR-0033). + +The generated packages (`/v.py`) are never edited by hand; the hand-written +modules at this level are listed in docs/process/git-workflow.md (counted in a size). +""" from importlib.metadata import version diff --git a/packages/contracts/src/orvex/contracts/binding.py b/packages/contracts/src/orvex/contracts/binding.py new file mode 100644 index 00000000..ffdb193a --- /dev/null +++ b/packages/contracts/src/orvex/contracts/binding.py @@ -0,0 +1,80 @@ +"""The two bindings an implementation is reached through (#486 G5): in process, and serialized +as an independent process would reach it. + +Through either binding only the contract's operations are callable, and an error crosses only +when it is the contract's error type with a code the operation declares +(INTERFACE_ERROR_UNDECLARED otherwise). Serialized, the arguments, the result and the error's +code, message and details each pass through the codec, so a live object is refused +(INTERFACE_VALUE_NOT_SERIALIZABLE) and nothing is shared with the caller. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from typing import Any + +from .codec import Codec +from .interface import InterfaceContract +from .interface_errors import InterfaceContractError + + +class Bound: + """An implementation of `contract`, reached through one binding.""" + + __slots__ = ("_codec", "_contract", "_implementation") + + def __init__( + self, contract: InterfaceContract, implementation: object, *, serialized: bool + ) -> None: + self._contract = contract + self._implementation = implementation + self._codec = Codec(contract.value_classes()) if serialized else None + + @property + def serialized(self) -> bool: + return self._codec is not None + + def __getattr__(self, name: str) -> Callable[..., Any]: + if name.startswith("_") or name not in {o.name for o in self._contract.operations}: + raise AttributeError(f"contract {self._contract.name} has no operation {name!r}") + return functools.partial(self.call, name) + + def call(self, operation: str, /, *arguments: object, **keywords: object) -> Any: # noqa: ANN401 + declared = self._contract.operation(operation) + method = getattr(self._implementation, operation, None) + if not callable(method): + raise InterfaceContractError( + "INTERFACE_OPERATION_MISSING", + f"the implementation of {self._contract.name} has no operation {operation}", + ) + codec = self._codec + if codec is not None: + arguments, keywords = _crossed(codec, (arguments, keywords)) + try: + result = method(*arguments, **keywords) + except self._contract.error_type as error: + code = getattr(error, "code", None) + if code not in declared.errors: + raise InterfaceContractError( + "INTERFACE_ERROR_UNDECLARED", + f"{self._contract.name}.{operation} raised {code!r}, which it does not declare", + ) from error + if codec is None: + raise + raise self._rebuilt(codec, error) from None + return result if codec is None else _crossed(codec, result) + + def _rebuilt(self, codec: Codec, error: Exception) -> Exception: + parts = tuple(getattr(error, name, None) for name in ("code", "message", "details")) + code, message, details = _crossed(codec, parts) + if not isinstance(message, str) or not isinstance(details, dict): + raise InterfaceContractError( + "INTERFACE_VALUE_NOT_SERIALIZABLE", + f"a {type(error).__name__} needs a message and details to cross the contract", + ) from error + return self._contract.error_type(code, message, details) + + +def _crossed(codec: Codec, value: object) -> Any: # noqa: ANN401 - the value decoded back + return codec.decode(codec.encode(value)) diff --git a/packages/contracts/src/orvex/contracts/codec.py b/packages/contracts/src/orvex/contracts/codec.py new file mode 100644 index 00000000..dd4b18d1 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/codec.py @@ -0,0 +1,160 @@ +"""The serializing codec (#486 G5): a value that crosses a contract, to bytes and back, exactly. + +Every value becomes tagged JSON: exact types survive (a tuple stays a tuple, `bool` is never an +`int`), a `Decimal` keeps its scale, an integer of any width stays exact, a `datetime` keeps its +zone (an IANA zone by its key) and its `fold`. Only the classes the contract declares +(`InterfaceContract.value_classes`) are built back, and only of exactly that class. Anything +else is refused with INTERFACE_VALUE_NOT_SERIALIZABLE: a live object never crosses. +""" + +from __future__ import annotations + +import base64 +import dataclasses +import json +from datetime import date, datetime, time, timedelta, timezone +from decimal import Decimal +from enum import Enum +from typing import Any +from zoneinfo import ZoneInfo + +from pydantic import BaseModel + +from .interface_errors import InterfaceContractError + +CODE = "INTERFACE_VALUE_NOT_SERIALIZABLE" + + +def qualified(kind: type) -> str: + return f"{kind.__module__}.{kind.__qualname__}" + + +def _refused(text: str) -> InterfaceContractError: + return InterfaceContractError(CODE, text) + + +class Codec: + """Encodes and decodes the values of one contract.""" + + def __init__(self, classes: frozenset[type]) -> None: + self._classes = {qualified(kind): kind for kind in classes} + + def encode(self, value: object) -> bytes: + return json.dumps(self._tree(value), ensure_ascii=False, separators=(",", ":")).encode() + + def decode(self, data: bytes) -> object: + try: + tree = json.loads(data) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise _refused(f"not an encoded value: {error}") from None + return self._value(tree) + + def _tree(self, value: object) -> Any: # noqa: ANN401 - a JSON tree + kind = type(value) + if value is None or kind in (bool, int, str): + return value + if kind is float: + return {"$float": repr(value)} + if kind is Decimal: + return {"$decimal": str(value)} + if kind is bytes and isinstance(value, bytes): + return {"$bytes": base64.b64encode(value).decode()} + if kind is datetime and isinstance(value, datetime): + return {"$datetime": value.replace(tzinfo=None).isoformat(), **_zone(value)} + if kind is date and isinstance(value, date): + return {"$date": value.isoformat()} + if kind is time and isinstance(value, time): + if value.tzinfo is not None and type(value.tzinfo) is not timezone: + raise _refused("a time of day carries a fixed offset or no zone") + return {"$time": value.isoformat()} + if kind is timedelta and isinstance(value, timedelta): + return {"$timedelta": [value.days, value.seconds, value.microseconds]} + if kind in (tuple, list) and isinstance(value, (tuple, list)): + return {f"${kind.__name__}": [self._tree(item) for item in value]} + if kind is dict and isinstance(value, dict): + if not all(type(key) is str for key in value): + raise _refused("only `str` keys cross a contract") + return {"$dict": {key: self._tree(item) for key, item in value.items()}} + return self._object(value) + + def _object(self, value: object) -> Any: # noqa: ANN401 - a JSON tree + kind = type(value) + if self._classes.get(qualified(kind)) is not kind: + raise _refused(f"a {qualified(kind)} cannot cross this contract") + name = qualified(kind) + if isinstance(value, Enum): + return {"$enum": name, "member": value.name} + if isinstance(value, BaseModel): + return {"$model": name, "json": value.model_dump_json()} + if not dataclasses.is_dataclass(value): + raise _refused(f"a {name} is not a value") + fields = {f.name: self._tree(getattr(value, f.name)) for f in dataclasses.fields(value)} + return {"$dataclass": name, "fields": fields} + + def _value(self, tree: Any) -> object: # noqa: ANN401 - a JSON tree + if tree is None or type(tree) in (bool, int, str): + return tree + if type(tree) is not dict or not tree: + raise _refused("not an encoded value") + tag = next(iter(tree)) + payload = tree[tag] + try: + return self._tagged(tag, payload, tree) + except (ValueError, TypeError, KeyError) as error: + raise _refused(f"not an encoded {tag}: {error}") from None + + def _tagged(self, tag: str, payload: Any, tree: dict[str, Any]) -> object: # noqa: ANN401 + if tag == "$float": + return float(payload) + if tag == "$decimal": + return Decimal(payload) + if tag == "$bytes": + return base64.b64decode(payload, validate=True) + if tag == "$datetime": + moment = datetime.fromisoformat(payload) + return moment.replace(tzinfo=_tzinfo(tree), fold=tree.get("fold", 0)) + if tag == "$date": + return date.fromisoformat(payload) + if tag == "$time": + return time.fromisoformat(payload) + if tag == "$timedelta": + days, seconds, microseconds = payload + return timedelta(days=days, seconds=seconds, microseconds=microseconds) + if tag == "$tuple": + return tuple(self._value(item) for item in payload) + if tag == "$list": + return [self._value(item) for item in payload] + if tag == "$dict": + return {key: self._value(item) for key, item in payload.items()} + kind = self._classes.get(payload) if type(payload) is str else None + if kind is None: + raise _refused(f"{tag} {payload!r} is not a class of this contract") + if tag == "$enum" and issubclass(kind, Enum): + return kind[tree["member"]] + if tag == "$model" and issubclass(kind, BaseModel): + return kind.model_validate_json(tree["json"]) + if tag == "$dataclass" and dataclasses.is_dataclass(kind): + return kind(**{name: self._value(item) for name, item in tree["fields"].items()}) + raise _refused(f"{tag} does not match {payload}") + + +def _zone(value: datetime) -> dict[str, object]: + zone = value.tzinfo + found: dict[str, object] = {"fold": value.fold} if value.fold else {} + if zone is None: + return found + if type(zone) is ZoneInfo and zone.key is not None: + return {**found, "zone": zone.key} + if type(zone) is timezone: + offset = zone.utcoffset(None) + return {**found, "offset": [offset.days, offset.seconds, offset.microseconds]} + raise _refused(f"a {qualified(type(zone))} zone cannot cross a contract") + + +def _tzinfo(tree: dict[str, Any]) -> ZoneInfo | timezone | None: + if "zone" in tree: + return ZoneInfo(tree["zone"]) + if "offset" in tree: + days, seconds, microseconds = tree["offset"] + return timezone(timedelta(days=days, seconds=seconds, microseconds=microseconds)) + return None diff --git a/packages/contracts/src/orvex/contracts/interface.py b/packages/contracts/src/orvex/contracts/interface.py new file mode 100644 index 00000000..d5eaee49 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/interface.py @@ -0,0 +1,234 @@ +"""The interface-contract format (#486 §5 step 1, ADR-0033): how a contract declares its +operations, inputs, outputs, errors, capabilities and version. + +An interface contract is a `typing.Protocol` whose public methods are the operations (their +annotations are the inputs and outputs), plus an `InterfaceContract` that gives the contract a +name and a `major.minor` version, groups the operations into capabilities, and lists the error +codes each operation may raise through the contract's error type (built as +`error_type(code, message, details)`). The declaration is checked when it is built: a refused +declaration never exists. +""" + +from __future__ import annotations + +import inspect +import re +import typing +from dataclasses import dataclass + +from .interface_errors import InterfaceContractError +from .serializable import crossing_problem, type_name + +NAME = re.compile(r"[a-z][a-z0-9]*(-[a-z0-9]+)*") +ERROR_CODE = re.compile(r"[A-Z][A-Z0-9]*(_[A-Z0-9]+)*") +VERSION = re.compile(r"([1-9][0-9]*)\.(0|[1-9][0-9]*)") +# A capability is either implemented in full or not provided: this code can never stand in for +# an operation a contract declares (#486 G1). +UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" +KINDS: dict[inspect._ParameterKind, str] = { + inspect.Parameter.POSITIONAL_ONLY: "positional-only", + inspect.Parameter.POSITIONAL_OR_KEYWORD: "positional-or-keyword", + inspect.Parameter.KEYWORD_ONLY: "keyword-only", +} + + +@dataclass(frozen=True, slots=True, order=True) +class Version: + """A contract version: within a major version the contract only grows (append-only).""" + + major: int + minor: int + + @classmethod + def parse(cls, text: str) -> Version: + match = VERSION.fullmatch(text) + if match is None: + raise InterfaceContractError( + "INTERFACE_VERSION_INVALID", f"{text!r} is not a version: expected MAJOR.MINOR" + ) + return cls(int(match[1]), int(match[2])) + + def provides(self, required: Version) -> bool: + """An implementation at this version serves a consumer that requires `required`.""" + return self.major == required.major and self.minor >= required.minor + + def __str__(self) -> str: + return f"{self.major}.{self.minor}" + + +@dataclass(frozen=True, slots=True) +class Operation: + """One operation: its method name, the capability it belongs to and the codes it raises.""" + + name: str + capability: str + errors: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class InterfaceContract: + """An interface contract: what any implementation must do, and nothing about how.""" + + name: str + version: Version + protocol: type + error_type: type[Exception] + capabilities: tuple[str, ...] + errors: tuple[str, ...] + operations: tuple[Operation, ...] + + def __post_init__(self) -> None: + _check(self) + + def operation(self, name: str) -> Operation: + for operation in self.operations: + if operation.name == name: + return operation + raise InterfaceContractError( + "INTERFACE_OPERATION_UNKNOWN", f"contract {self.name} has no operation {name!r}" + ) + + def operations_of(self, capability: str) -> tuple[Operation, ...]: + return tuple(o for o in self.operations if o.capability == capability) + + def signature(self, name: str) -> inspect.Signature: + """The operation's signature without `self`, its annotations resolved.""" + return _signature(self.protocol, self.operation(name).name) + + def value_classes(self) -> frozenset[type]: + """The enums, models and dataclasses that may cross this contract.""" + classes: set[type] = set() + for operation in self.operations: + for annotation in _annotations(self.signature(operation.name)): + crossing_problem(annotation, classes) + return frozenset(classes) + + def describe(self) -> dict[str, object]: + """The contract as plain data: what the compatibility check compares (#486 G3).""" + operations: dict[str, object] = {} + for operation in self.operations: + signature = self.signature(operation.name) + operations[operation.name] = { + "capability": operation.capability, + "errors": sorted(operation.errors), + "parameters": [ + { + "name": parameter.name, + "kind": KINDS[parameter.kind], + "type": type_name(parameter.annotation), + "default": parameter.default is not inspect.Parameter.empty, + } + for parameter in signature.parameters.values() + ], + "returns": type_name(signature.return_annotation), + } + return { + "name": self.name, + "version": str(self.version), + "capabilities": sorted(self.capabilities), + "errors": sorted(self.errors), + "operations": operations, + } + + +def _invalid(contract: InterfaceContract, text: str) -> InterfaceContractError: + return InterfaceContractError( + "INTERFACE_CONTRACT_INVALID", + f"contract {contract.name!r}: {text}", + {"contract": contract.name}, + ) + + +def _check(contract: InterfaceContract) -> None: + if not NAME.fullmatch(contract.name): + raise _invalid(contract, "a name is lowercase-kebab-case") + if not (isinstance(contract.error_type, type) and issubclass(contract.error_type, Exception)): + raise _invalid(contract, "the error type must be an exception class") + if UNSUPPORTED_OPERATION in contract.errors or any( + UNSUPPORTED_OPERATION in operation.errors for operation in contract.operations + ): + raise InterfaceContractError( + "INTERFACE_UNSUPPORTED_OPERATION_DECLARED", + f"contract {contract.name!r}: {UNSUPPORTED_OPERATION} can never stand in for an " + "operation; an implementation provides a capability in full or not at all", + {"contract": contract.name}, + ) + _check_names(contract) + _check_operations(contract) + + +def _check_names(contract: InterfaceContract) -> None: + if len(set(contract.capabilities)) != len(contract.capabilities) or not all( + NAME.fullmatch(capability) for capability in contract.capabilities + ): + raise _invalid(contract, "capabilities are unique and lowercase-kebab-case") + if len(set(contract.errors)) != len(contract.errors) or not all( + ERROR_CODE.fullmatch(code) for code in contract.errors + ): + raise _invalid(contract, "error codes are unique and UPPER_SNAKE_CASE") + names = [operation.name for operation in contract.operations] + if not names or len(set(names)) != len(names): + raise _invalid(contract, "a contract has at least one operation, each named once") + for operation in contract.operations: + if operation.capability not in contract.capabilities: + raise _invalid( + contract, f"{operation.name}: capability {operation.capability!r} is undeclared" + ) + unknown = sorted(set(operation.errors) - set(contract.errors)) + if unknown: + raise _invalid( + contract, f"{operation.name}: error codes {unknown} are not the contract's" + ) + empty = [c for c in contract.capabilities if not contract.operations_of(c)] + if empty: + raise _invalid(contract, f"capabilities {empty} have no operation") + public = sorted( + name + for name, value in vars(contract.protocol).items() + if not name.startswith("_") and (callable(value) or isinstance(value, property)) + ) + if public != sorted(names): + raise _invalid( + contract, + f"the protocol's public methods {public} must be the operations {sorted(names)}", + ) + + +def _check_operations(contract: InterfaceContract) -> None: + for operation in contract.operations: + member = vars(contract.protocol)[operation.name] + if not inspect.isfunction(member): + raise _invalid(contract, f"{operation.name}: an operation is a plain method") + signature = _signature(contract.protocol, operation.name) + for parameter in signature.parameters.values(): + if parameter.kind not in KINDS: + raise _invalid(contract, f"{operation.name}: *{parameter.name} cannot be described") + if parameter.annotation is inspect.Parameter.empty: + raise _invalid(contract, f"{operation.name}: {parameter.name} has no annotation") + if signature.return_annotation is inspect.Signature.empty: + raise _invalid(contract, f"{operation.name}: the result has no annotation") + for annotation in _annotations(signature): + problem = crossing_problem(annotation, set()) + if problem is not None: + raise InterfaceContractError( + "INTERFACE_TYPE_NOT_SERIALIZABLE", + f"contract {contract.name!r}, {operation.name}: {problem}", + {"contract": contract.name, "operation": operation.name}, + ) + + +def _signature(protocol: type, name: str) -> inspect.Signature: + method = vars(protocol)[name] + hints = typing.get_type_hints(method) + signature = inspect.signature(method) + parameters = [ + parameter.replace(annotation=hints.get(parameter.name, inspect.Parameter.empty)) + for parameter in list(signature.parameters.values())[1:] + ] + returns = hints.get("return", inspect.Signature.empty) + return signature.replace(parameters=parameters, return_annotation=returns) + + +def _annotations(signature: inspect.Signature) -> list[object]: + found: list[object] = [parameter.annotation for parameter in signature.parameters.values()] + return [*found, signature.return_annotation] diff --git a/packages/contracts/src/orvex/contracts/interface_errors.py b/packages/contracts/src/orvex/contracts/interface_errors.py new file mode 100644 index 00000000..6ef17be8 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/interface_errors.py @@ -0,0 +1,22 @@ +"""The error of the interface-contract machinery: a stable code, a message and details. + +Codes: INTERFACE_CONTRACT_INVALID, INTERFACE_VERSION_INVALID, INTERFACE_TYPE_NOT_SERIALIZABLE, +INTERFACE_UNSUPPORTED_OPERATION_DECLARED, INTERFACE_OPERATION_UNKNOWN, +INTERFACE_OPERATION_MISSING, INTERFACE_VALUE_NOT_SERIALIZABLE, INTERFACE_ERROR_UNDECLARED, +KIT_INVALID, KIT_CASE_COUNT_CHANGED, KIT_OPERATION_UNCOVERED, KIT_FAILED and +MODULE_MANIFEST_INVALID. +""" + +from __future__ import annotations + +from collections.abc import Mapping + + +class InterfaceContractError(Exception): + """A declaration, a value or a run that breaks the interface-contract rules (ADR-0033).""" + + def __init__(self, code: str, message: str, details: Mapping[str, str] | None = None) -> None: + super().__init__(f"{code}: {message}") + self.code = code + self.message = message + self.details = dict(details or {}) diff --git a/packages/contracts/src/orvex/contracts/kit.py b/packages/contracts/src/orvex/contracts/kit.py new file mode 100644 index 00000000..ab46c16c --- /dev/null +++ b/packages/contracts/src/orvex/contracts/kit.py @@ -0,0 +1,73 @@ +"""The conformance-kit format (#486 G1): the cases every implementation of a contract must pass, +with their recorded count and no skip list. + +A kit belongs to one interface contract. Each case has a stable id, the operation it exercises, +its expectation in words (the guarantee the compatibility check holds to, #486 G3) and a check +that receives the implementation through a binding and raises when the guarantee fails. The +kit records how many cases it has; a kit whose cases differ from that count, or that leaves an +operation without a case, is refused when it is built. +""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from dataclasses import dataclass + +from .binding import Bound +from .interface import InterfaceContract +from .interface_errors import InterfaceContractError + +CASE_ID = re.compile(r"[a-z][a-z0-9]*([.-][a-z0-9]+)*") + + +@dataclass(frozen=True, slots=True) +class KitCase: + id: str + operation: str + expectation: str + check: Callable[[Bound], None] + + +@dataclass(frozen=True, slots=True) +class Kit: + contract: InterfaceContract + recorded: int + cases: tuple[KitCase, ...] + + def __post_init__(self) -> None: + name = self.contract.name + operations = {operation.name for operation in self.contract.operations} + seen: set[str] = set() + for case in self.cases: + problem = ( + "its id is not lowercase, dotted or kebab-case" + if not CASE_ID.fullmatch(case.id) + else "its id is used twice" + if case.id in seen + else f"operation {case.operation!r} is not the contract's" + if case.operation not in operations + else "it states no expectation" + if not case.expectation.strip() + else None + ) + if problem is not None: + raise InterfaceContractError( + "KIT_INVALID", f"kit {name}, case {case.id!r}: {problem}" + ) + seen.add(case.id) + uncovered = sorted(operations - {case.operation for case in self.cases}) + if uncovered: + raise InterfaceContractError( + "KIT_OPERATION_UNCOVERED", f"kit {name}: operations {uncovered} have no case" + ) + if len(self.cases) != self.recorded: + raise InterfaceContractError( + "KIT_CASE_COUNT_CHANGED", + f"kit {name} records {self.recorded} cases and has {len(self.cases)}", + ) + + def describe(self) -> dict[str, object]: + """The kit as plain data: what the compatibility check compares (#486 G3).""" + cases = {c.id: {"operation": c.operation, "expectation": c.expectation} for c in self.cases} + return {"recorded": self.recorded, "cases": cases} diff --git a/packages/contracts/src/orvex/contracts/kit_run.py b/packages/contracts/src/orvex/contracts/kit_run.py new file mode 100644 index 00000000..4a043517 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/kit_run.py @@ -0,0 +1,83 @@ +"""Running a conformance kit (#486 G1, G5): every case, in process and serialized, each on a fresh +implementation, with nothing skipped and the same results through both bindings.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from .binding import Bound +from .interface_errors import InterfaceContractError +from .kit import Kit + + +@dataclass(frozen=True, slots=True) +class CaseResult: + id: str + passed: bool + detail: str + + +@dataclass(frozen=True, slots=True) +class KitReport: + contract: str + serialized: bool + results: tuple[CaseResult, ...] + + +def binding_name(serialized: bool) -> str: + return "serialized" if serialized else "in process" + + +def run_kit(kit: Kit, make: Callable[[], object], *, serialized: bool) -> KitReport: + results: list[CaseResult] = [] + for case in kit.cases: + bound = Bound(kit.contract, make(), serialized=serialized) + try: + case.check(bound) + except KeyboardInterrupt, SystemExit: + raise + except BaseException as error: # noqa: BLE001 - a failure or a skip: the kit has no skips + results.append(CaseResult(case.id, False, f"{type(error).__name__}: {error}")) + else: + results.append(CaseResult(case.id, True, "")) + return KitReport(kit.contract.name, serialized, tuple(results)) + + +def kit_problems(kit: Kit, report: KitReport) -> list[str]: + """Every way `report` falls short of the whole kit passing.""" + binding = binding_name(report.serialized) + ran = [result.id for result in report.results] + problems = [ + f"KIT_RUN_INCOMPLETE: {case.id} did not run ({binding})" + for case in kit.cases + if case.id not in ran + ] + if len(ran) != kit.recorded and not problems: + problems.append( + f"KIT_RUN_INCOMPLETE: {len(ran)} results for {kit.recorded} cases ({binding})" + ) + problems += [ + f"KIT_CASE_FAILED: {result.id} ({binding}): {result.detail}" + for result in report.results + if not result.passed + ] + return problems + + +def verify_kit(kit: Kit, make: Callable[[], object]) -> tuple[KitReport, KitReport]: + """Run the whole kit in process and serialized; raise KIT_FAILED unless every case passes + through both bindings with the same results.""" + in_process = run_kit(kit, make, serialized=False) + serialized = run_kit(kit, make, serialized=True) + problems = kit_problems(kit, in_process) + kit_problems(kit, serialized) + outcomes = [[(r.id, r.passed) for r in report.results] for report in (in_process, serialized)] + if outcomes[0] != outcomes[1]: + problems.insert( + 0, "KIT_BINDINGS_DIFFER: the serialized run differs from the run in process" + ) + if problems: + raise InterfaceContractError( + "KIT_FAILED", "; ".join(problems), {"contract": kit.contract.name} + ) + return in_process, serialized diff --git a/packages/contracts/src/orvex/contracts/manifest.py b/packages/contracts/src/orvex/contracts/manifest.py new file mode 100644 index 00000000..1d328483 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/manifest.py @@ -0,0 +1,168 @@ +"""The module manifest `module.toml` (#486 §2.5): the interface contracts a module provides and +requires, each at a version with its capabilities, and the distributions it depends on. + + module = "store" + + [dependencies] + runtime = ["duckdb", "pyarrow"] # what its code imports, equal to its distribution's + tests = ["pytest"] # what its own tests import besides those + + [[provides]] # one per contract it implements + contract = "store" + version = "1.0" + capabilities = ["read", "write"] + implementation = "Store" # an attribute of the module's `api` package, nothing else + + [[requires]] # one per contract it consumes + contract = "security-context" + version = "1.0" + capabilities = ["read"] + +Anything else, missing or misspelt is refused with MODULE_MANIFEST_INVALID. +""" + +from __future__ import annotations + +import re +import tomllib +from dataclasses import dataclass +from typing import Any + +from .interface import NAME, Version +from .interface_errors import InterfaceContractError + +MODULE = re.compile(r"[a-z]+") +DISTRIBUTION = re.compile(r"[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?") +IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") + + +@dataclass(frozen=True, slots=True) +class Provided: + contract: str + version: Version + capabilities: tuple[str, ...] + implementation: str + + +@dataclass(frozen=True, slots=True) +class Required: + contract: str + version: Version + capabilities: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class Manifest: + module: str + provides: tuple[Provided, ...] + requires: tuple[Required, ...] + runtime: tuple[str, ...] + tests: tuple[str, ...] + + +def canonical(distribution: str) -> str: + """A distribution name in its normalized form (PEP 503).""" + return re.sub(r"[-_.]+", "-", distribution).lower() + + +def parse_manifest(text: str, source: str) -> Manifest: + """The manifest in `text`; `source` names the module it was read from, for the error.""" + try: + data = tomllib.loads(text) + return _manifest(data) + except (tomllib.TOMLDecodeError, ValueError, InterfaceContractError) as error: + raise InterfaceContractError( + "MODULE_MANIFEST_INVALID", f"{source}/module.toml: {error}", {"module": source} + ) from None + + +def _keys( + table: object, where: str, keys: set[str], optional: frozenset[str] = frozenset() +) -> dict[str, Any]: + if not isinstance(table, dict): + raise ValueError(f"{where} must be a table") + missing = sorted(keys - set(table)) + unknown = sorted(set(table) - keys - optional) + if missing or unknown: + raise ValueError(f"{where}: missing {missing}, unknown {unknown}") + return table + + +def _strings( + value: object, where: str, pattern: re.Pattern[str], empty: bool = True +) -> tuple[str, ...]: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{where} must be a list of strings") + if not empty and not value: + raise ValueError(f"{where} may not be empty") + if not all(pattern.fullmatch(item) for item in value): + raise ValueError(f"{where}: {value} does not match {pattern.pattern}") + return tuple(value) + + +def _distributions(value: object, where: str) -> tuple[str, ...]: + names = [canonical(name) for name in _strings(value, where, DISTRIBUTION)] + if len(set(names)) != len(names): + raise ValueError(f"{where} names a distribution twice") + return tuple(sorted(names)) + + +def _capabilities(value: object, where: str) -> tuple[str, ...]: + found = _strings(value, where, NAME, empty=False) + if len(set(found)) != len(found): + raise ValueError(f"{where} names a capability twice") + return found + + +def _contract(entry: dict[str, Any], where: str) -> tuple[str, Version, tuple[str, ...]]: + contract = entry["contract"] + if not isinstance(contract, str) or not NAME.fullmatch(contract): + raise ValueError(f"{where}: contract {contract!r} is not lowercase-kebab-case") + version = entry["version"] + if not isinstance(version, str): + raise ValueError(f"{where}: version must be a string") + return ( + contract, + Version.parse(version), + _capabilities(entry["capabilities"], f"{where}.capabilities"), + ) + + +def _manifest(data: dict[str, Any]) -> Manifest: + _keys(data, "the manifest", {"module", "dependencies"}, frozenset({"provides", "requires"})) + module = data["module"] + if not isinstance(module, str) or not MODULE.fullmatch(module): + raise ValueError(f"module {module!r} is not a module name") + dependencies = _keys(data["dependencies"], "[dependencies]", {"runtime", "tests"}) + provides = [] + for index, entry in enumerate(_tables(data.get("provides", []), "provides")): + where = f"provides[{index}]" + _keys(entry, where, {"contract", "version", "capabilities", "implementation"}) + implementation = entry["implementation"] + if not isinstance(implementation, str) or not IDENTIFIER.fullmatch(implementation): + raise ValueError( + f"{where}: implementation is one attribute of the module's api package" + ) + provides.append(Provided(*_contract(entry, where), implementation)) + requires = [] + for index, entry in enumerate(_tables(data.get("requires", []), "requires")): + where = f"requires[{index}]" + _keys(entry, where, {"contract", "version", "capabilities"}) + requires.append(Required(*_contract(entry, where))) + for kind, entries in (("provides", provides), ("requires", requires)): + contracts = [entry.contract for entry in entries] + if len(set(contracts)) != len(contracts): + raise ValueError(f"{kind} names a contract twice") + return Manifest( + module, + tuple(provides), + tuple(requires), + _distributions(dependencies["runtime"], "dependencies.runtime"), + _distributions(dependencies["tests"], "dependencies.tests"), + ) + + +def _tables(value: object, where: str) -> list[dict[str, Any]]: + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise ValueError(f"{where} must be an array of tables") + return value diff --git a/packages/contracts/src/orvex/contracts/module.toml b/packages/contracts/src/orvex/contracts/module.toml new file mode 100644 index 00000000..f648688c --- /dev/null +++ b/packages/contracts/src/orvex/contracts/module.toml @@ -0,0 +1,7 @@ +# The manifest of this module (#486 §2.5, orvex.contracts.manifest): it implements no interface +# contract and requires none; the contract types depend on pydantic. +module = "contracts" + +[dependencies] +runtime = ["pydantic"] +tests = ["pytest"] diff --git a/packages/contracts/src/orvex/contracts/registry.py b/packages/contracts/src/orvex/contracts/registry.py new file mode 100644 index 00000000..b0fdb362 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/registry.py @@ -0,0 +1,48 @@ +"""The interface contracts and kits a contracts package ships, found by their place: +`..interface` holds `InterfaceContract` values and `..kit` holds +`Kit` values (ADR-0033).""" + +from __future__ import annotations + +import importlib +import importlib.util +import pkgutil +from types import ModuleType + +from .interface import InterfaceContract +from .interface_errors import InterfaceContractError +from .kit import Kit + + +def interface_contracts(package: ModuleType) -> dict[str, InterfaceContract]: + return _declared(package, "interface", InterfaceContract) + + +def kits(package: ModuleType) -> dict[str, Kit]: + return _declared(package, "kit", Kit) + + +def _name(value: InterfaceContract | Kit) -> str: + return value.name if isinstance(value, InterfaceContract) else value.contract.name + + +def _declared[Declared: (InterfaceContract, Kit)]( + package: ModuleType, place: str, kind: type[Declared] +) -> dict[str, Declared]: + found: dict[str, Declared] = {} + for entry in sorted(pkgutil.iter_modules(package.__path__), key=lambda e: e.name): + name = f"{package.__name__}.{entry.name}.{place}" + if not entry.ispkg or importlib.util.find_spec(name) is None: + continue + for value in vars(importlib.import_module(name)).values(): + if not isinstance(value, kind): + continue + key = _name(value) + if key in found and found[key] is not value: + raise InterfaceContractError( + "INTERFACE_CONTRACT_INVALID", + f"two {place} declarations name the contract {key!r}", + {"contract": key}, + ) + found[key] = value + return found diff --git a/packages/contracts/src/orvex/contracts/serializable.py b/packages/contracts/src/orvex/contracts/serializable.py new file mode 100644 index 00000000..33d10c15 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/serializable.py @@ -0,0 +1,105 @@ +"""Which annotated types may cross an interface contract (#486 G5): plain values, never live +objects, so every operation can later run in another process (#156). + +A value type is one of `SCALARS`, a `Literal` of booleans, integers, strings or `None`, a +union of value types, a parameterized `tuple`, `list` or `dict` with `str` keys, an `Enum`, a +pydantic model (the generated contract types) or a frozen dataclass whose fields are all value +types. Anything else (`Any`, `object`, callables, iterators, locks, connections, a mutable +dataclass, a bare `tuple`) is refused. +""" + +from __future__ import annotations + +import dataclasses +import types +import typing +from datetime import date, datetime, time, timedelta +from decimal import Decimal +from enum import Enum + +from pydantic import BaseModel + +SCALARS: tuple[type, ...] = ( + type(None), + bool, + int, + float, + str, + bytes, + Decimal, + datetime, + date, + time, + timedelta, +) +LITERAL_TYPES = (bool, int, str, type(None)) +SEQUENCES = (tuple, list) + + +def type_name(annotation: object) -> str: + """A stable text for an annotation: `int`, `tuple[str, ...]`, `a.b.Reading | None`.""" + if annotation is type(None) or annotation is None: + return "None" + if annotation is Ellipsis: + return "..." + origin = typing.get_origin(annotation) + arguments = typing.get_args(annotation) + if origin in (typing.Union, types.UnionType): + return " | ".join(type_name(argument) for argument in arguments) + if origin is typing.Literal: + return f"Literal[{', '.join(repr(argument) for argument in arguments)}]" + if origin is not None: + inner = ", ".join(type_name(argument) for argument in arguments) + return f"{type_name(origin)}[{inner}]" + if isinstance(annotation, type): + if annotation.__module__ == "builtins": + return annotation.__qualname__ + return f"{annotation.__module__}.{annotation.__qualname__}" + return repr(annotation) + + +def crossing_problem(annotation: object, classes: set[type]) -> str | None: + """Why `annotation` may not cross a contract, or None; the classes it reaches (enums, + models, dataclasses) are added to `classes`.""" + if annotation in SCALARS or annotation is None: + return None + origin = typing.get_origin(annotation) + arguments = typing.get_args(annotation) + if origin in (typing.Union, types.UnionType): + return _first(crossing_problem(argument, classes) for argument in arguments) + if origin is typing.Literal: + if all(type(argument) in LITERAL_TYPES for argument in arguments): + return None + return f"{type_name(annotation)} holds a value that is not a plain literal" + if origin in SEQUENCES: + items = [argument for argument in arguments if argument is not Ellipsis] + if not items: + return f"{type_name(annotation)} does not say what it holds" + return _first(crossing_problem(item, classes) for item in items) + if origin is dict: + if not arguments or arguments[0] is not str: + return f"{type_name(annotation)}: only `str` keys may cross" + return crossing_problem(arguments[1], classes) + if origin is None and isinstance(annotation, type): + return _class_problem(annotation, classes) + return f"{type_name(annotation)} is not a value type: a live object cannot cross a contract" + + +def _class_problem(annotation: type, classes: set[type]) -> str | None: + if annotation in classes: + return None + if issubclass(annotation, (Enum, BaseModel)): + classes.add(annotation) + return None + if dataclasses.is_dataclass(annotation): + if not annotation.__dataclass_params__.frozen: # type: ignore[attr-defined] + return f"{type_name(annotation)} is a mutable dataclass" + classes.add(annotation) + hints = typing.get_type_hints(annotation) + fields = (hints[field.name] for field in dataclasses.fields(annotation)) + return _first(crossing_problem(field, classes) for field in fields) + return f"{type_name(annotation)} is not a value type: a live object cannot cross a contract" + + +def _first(problems: typing.Iterable[str | None]) -> str | None: + return next((problem for problem in problems if problem is not None), None) diff --git a/packages/server/src/orvex/server/assembly.py b/packages/server/src/orvex/server/assembly.py new file mode 100644 index 00000000..db190e04 --- /dev/null +++ b/packages/server/src/orvex/server/assembly.py @@ -0,0 +1,163 @@ +"""The assembly check (#486 G1): before the server composes the modules, every interface contract +a module requires has exactly one installed implementation, at a compatible version, with every +required capability and every operation of what it claims to provide. + +Every installed module may ship `module.toml` (`orvex.contracts.manifest`). An implementation +is loaded only from the module's `api` package, the one place this file imports a module by. +`UNSUPPORTED_OPERATION` cannot stand in for a capability: the contract format refuses to +declare it, and an operation the implementation lacks refuses the assembly. Any problem stops +the server before it opens a project or binds a port (exit status 3). +""" + +from __future__ import annotations + +import importlib +from collections.abc import Mapping, Sequence +from importlib.resources import files + +from orvex.contracts.interface import InterfaceContract +from orvex.contracts.manifest import Manifest, Provided, parse_manifest +from orvex.contracts.registry import interface_contracts +from orvex.server.identity import Identity +from orvex.server.versions import module_versions + + +class AssemblyRefused(Exception): + """The chosen implementations cannot serve their consumers; the server must not start.""" + + code = "ASSEMBLY_REFUSED" + + def __init__(self, problems: list[str]) -> None: + super().__init__(f"{self.code}: " + "; ".join(problems)) + self.problems = problems + + +def assembly_problems( + manifests: Sequence[Manifest], + contracts: Mapping[str, InterfaceContract], + implementations: Mapping[tuple[str, str], object], +) -> list[str]: + """Every reason the modules cannot be assembled; empty when they can.""" + problems: list[str] = [] + providers: dict[str, list[tuple[str, Provided]]] = {} + for manifest in manifests: + for provided in manifest.provides: + providers.setdefault(provided.contract, []).append((manifest.module, provided)) + problems += _claim_problems(manifest.module, provided, contracts, implementations) + for manifest in manifests: + for required in manifest.requires: + where = f"{manifest.module} requires {required.contract} {required.version}" + contract = contracts.get(required.contract) + if contract is None: + problems.append(f"ASSEMBLY_CONTRACT_UNKNOWN: {where}: no such interface contract") + continue + unknown = sorted(set(required.capabilities) - set(contract.capabilities)) + if unknown: + problems.append( + f"ASSEMBLY_CAPABILITY_UNKNOWN: {where}: {unknown} are not its capabilities" + ) + offered = providers.get(required.contract, []) + if not offered: + problems.append( + f"ASSEMBLY_NO_IMPLEMENTATION: {where}: no installed module provides it" + ) + continue + if len(offered) > 1: + names = ", ".join(module for module, _ in offered) + problems.append(f"ASSEMBLY_AMBIGUOUS_IMPLEMENTATION: {where}: provided by {names}") + continue + module, provided = offered[0] + if not provided.version.provides(required.version): + problems.append( + f"ASSEMBLY_VERSION_INCOMPATIBLE: {where}: {module} provides {provided.version}" + ) + missing = sorted(set(required.capabilities) - set(provided.capabilities) - set(unknown)) + if missing: + problems.append( + f"ASSEMBLY_CAPABILITY_MISSING: {where}: {module} does not provide {missing}" + ) + return problems + + +def _claim_problems( + module: str, + provided: Provided, + contracts: Mapping[str, InterfaceContract], + implementations: Mapping[tuple[str, str], object], +) -> list[str]: + """What is untrue in one module's claim to provide a contract.""" + where = f"{module} provides {provided.contract} {provided.version}" + contract = contracts.get(provided.contract) + if contract is None: + return [f"ASSEMBLY_CONTRACT_UNKNOWN: {where}: no such interface contract"] + if not contract.version.provides(provided.version): + return [ + f"ASSEMBLY_VERSION_INCOMPATIBLE: {where}: the installed contract is {contract.version}" + ] + unknown = sorted(set(provided.capabilities) - set(contract.capabilities)) + if unknown: + return [f"ASSEMBLY_CAPABILITY_UNKNOWN: {where}: {unknown} are not its capabilities"] + implementation = implementations.get((module, provided.contract)) + missing = [ + operation.name + for capability in provided.capabilities + for operation in contract.operations_of(capability) + if not callable(getattr(implementation, operation.name, None)) + ] + if missing: + return [f"ASSEMBLY_OPERATION_MISSING: {where}: {provided.implementation} lacks {missing}"] + return [] + + +def check_assembly( + manifests: Sequence[Manifest], + contracts: Mapping[str, InterfaceContract], + implementations: Mapping[tuple[str, str], object], +) -> None: + problems = assembly_problems(manifests, contracts, implementations) + if problems: + raise AssemblyRefused(problems) + + +def installed_manifests(identity: Identity) -> tuple[Manifest, ...]: + """The `module.toml` of every installed module that ships one.""" + found = [] + for module, _ in module_versions(identity): + resource = files(f"{identity.python_namespace}.{module}").joinpath("module.toml") + if resource.is_file(): + found.append(parse_manifest(resource.read_text("utf-8"), module)) + return tuple(found) + + +def load_implementations( + namespace: str, manifests: Sequence[Manifest] +) -> dict[tuple[str, str], object]: + """Each provided implementation, taken from its module's `api` package and nowhere else.""" + loaded: dict[tuple[str, str], object] = {} + problems = [] + for manifest in manifests: + for provided in manifest.provides: + package = f"{namespace}.{manifest.module}.api" + try: + implementation = getattr(importlib.import_module(package), provided.implementation) + except (ModuleNotFoundError, AttributeError) as error: + problems.append( + f"ASSEMBLY_IMPLEMENTATION_MISSING: {manifest.module} provides " + f"{provided.contract} as {package}.{provided.implementation}: {error}" + ) + continue + loaded[(manifest.module, provided.contract)] = implementation + if problems: + raise AssemblyRefused(problems) + return loaded + + +def check_installed_assembly(identity: Identity) -> None: + """The check run before the server starts, on the installed modules.""" + manifests = installed_manifests(identity) + contracts_package = importlib.import_module(f"{identity.python_namespace}.contracts") + check_assembly( + manifests, + interface_contracts(contracts_package), + load_implementations(identity.python_namespace, manifests), + ) diff --git a/packages/server/src/orvex/server/cli.py b/packages/server/src/orvex/server/cli.py index 8ac5316d..17fe107e 100644 --- a/packages/server/src/orvex/server/cli.py +++ b/packages/server/src/orvex/server/cli.py @@ -18,9 +18,11 @@ from types import FrameType import uvicorn +from orvex.contracts.interface_errors import InterfaceContractError from orvex.server import clickhouse_engines, development_switch, existing_lakes from orvex.server.admin_command import add_admin_parser, admin from orvex.server.app import LocalSemantic, create_app +from orvex.server.assembly import AssemblyRefused, check_installed_assembly from orvex.server.clickhouse_engines import ClickHouseEngines, clickhouse_connections from orvex.server.evidence_command import evidence from orvex.server.existing_lakes import ExistingLakes, ducklake_connections, memory_share @@ -331,6 +333,11 @@ def main(argv: Sequence[str] | None = None) -> int: except ContractVersionIncompatible as error: print(error, file=sys.stderr) return 3 + try: + check_installed_assembly(identity) + except (AssemblyRefused, InterfaceContractError) as error: + print(error, file=sys.stderr) + return 3 configure_logging() registry = ProjectRegistry(instance_directory(identity)) diff --git a/tools/checks/boundaries.py b/tools/checks/boundaries.py index ac0895b1..ba1d3d61 100644 --- a/tools/checks/boundaries.py +++ b/tools/checks/boundaries.py @@ -5,6 +5,8 @@ `.contracts`, the standard library and third-party packages; `server` composes all. 2. MATERIALIZATION_IN_PYTHON: `tools/checks/materialization.py`. 3. DATA_READ_OUTSIDE_DUCKDB: `tools/checks/data_readers.py`. +4. CONSUMER_BOUNDARY_VIOLATION: `tools/checks/consumers.py`, reported without failing until + #486 step 7 makes it blocking and removes the `server` exemption of rule 1. """ from __future__ import annotations @@ -14,6 +16,7 @@ from pathlib import Path from tools.checks.architecture import AllowListError, Finding, product_files +from tools.checks.consumers import consumer_scan, consumer_summary from tools.checks.data_readers import ( READER_ALLOW_LIST, load_reader_allow_list, @@ -117,6 +120,10 @@ def run(root: Path, materialization_list: Path, reader_list: Path) -> int: for finding in findings: print(finding) print(f"boundaries: {len(findings)} finding(s)") + consumers = consumer_scan(root) + for finding in consumers: + print(finding) + print(consumer_summary(consumers)) return 1 if findings else 0 diff --git a/tools/checks/consumers.py b/tools/checks/consumers.py new file mode 100644 index 00000000..10ddfb35 --- /dev/null +++ b/tools/checks/consumers.py @@ -0,0 +1,112 @@ +"""Consumer boundaries (#486 G2): `server`, the conformance runners, `tools/` and the repository +tests import only `.contracts`; `server/assembly.py` may also import the modules' +`api` packages, and the server's own files the server. + +CONSUMER_BOUNDARY_VIOLATION: any other import of a module, written as `import`, `from … import` +or a constant `importlib.import_module(…)` / `__import__(…)`. In report mode until #486 step 7: +`python -m tools.checks.boundaries` prints every violation and a summary per area, and fails +only on the module boundaries. +""" + +from __future__ import annotations + +import ast +from collections import Counter +from pathlib import Path + +from tools.checks.architecture import Finding +from tools.repository import MODULES, identity + +CODE = "CONSUMER_BOUNDARY_VIOLATION" +# Each consumer area, by the start of its paths. +AREAS = { + "server": "packages/server/", + "runners": "conformance/runners/", + "tools": "tools/", + "tests": "tests/", +} +SHARED = "contracts" +PUBLIC = "api" +DYNAMIC = {"import_module", "__import__"} + + +def assembly(namespace: str) -> str: + return f"packages/server/src/{namespace}/server/assembly.py" + + +def _imports(tree: ast.AST) -> list[tuple[str, int]]: + found: list[tuple[str, int]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.extend((alias.name, node.lineno) for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.append((node.module, node.lineno)) + found.extend((f"{node.module}.{alias.name}", node.lineno) for alias in node.names) + elif isinstance(node, ast.Call) and node.args: + function = node.func + name = ( + function.attr + if isinstance(function, ast.Attribute) + else getattr(function, "id", "") + ) + first = node.args[0] + if name in DYNAMIC and isinstance(first, ast.Constant) and isinstance(first.value, str): + found.append((first.value, node.lineno)) + return found + + +def _area(relative: str) -> str | None: + return next((area for area, start in AREAS.items() if relative.startswith(start)), None) + + +def consumer_findings(tree: ast.AST, namespace: str, relative: str) -> list[Finding]: + """The internal imports of one consumer file, one per module and line.""" + area = _area(relative) + if area is None: + return [] + findings: list[Finding] = [] + reported: set[tuple[str, int]] = set() + for name, line in _imports(tree): + parts = name.split(".") + if len(parts) < 2 or parts[0] != namespace or parts[1] not in MODULES: + continue + target = parts[1] + allowed = ( + target == SHARED + or (area == "server" and target == "server") + or (relative == assembly(namespace) and len(parts) > 2 and parts[2] == PUBLIC) + ) + if allowed or (target, line) in reported: + continue + reported.add((target, line)) + findings.append( + Finding(CODE, relative, line, f"imports '{name}', an internal of '{target}'") + ) + return findings + + +def consumer_scan(root: Path) -> list[Finding]: + namespace = str(identity(root)["python_namespace"]) + findings: list[Finding] = [] + for start in AREAS.values(): + base = root / start + if not base.is_dir(): + continue + for path in sorted(base.rglob("*.py")): + relative = path.relative_to(root).as_posix() + if any(part in {"build", "__pycache__", "node_modules"} for part in path.parts): + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=relative) + findings += consumer_findings(tree, namespace, relative) + return findings + + +def consumer_summary(findings: list[Finding]) -> str: + """One line: how many violations remain, in how many files, per area.""" + files = {finding.path for finding in findings} + per_area = Counter(_area(path) for path in files) + areas = ", ".join(f"{area} {per_area.get(area, 0)}" for area in AREAS) + return ( + f"consumer boundaries (report mode until #486 step 7): {len(findings)} finding(s) " + f"in {len(files)} file(s): {areas}" + ) diff --git a/tools/checks/module_dependencies.py b/tools/checks/module_dependencies.py new file mode 100644 index 00000000..3bf2beae --- /dev/null +++ b/tools/checks/module_dependencies.py @@ -0,0 +1,196 @@ +"""Inside a module's clean environment (#486 G4): the distributions its code imports equal those +it declares, and its own tests import no repository helper. + + /bin/python -I tools/checks/module_dependencies.py NAMESPACE DISTRIBUTION TESTS + +Run by `tools/ci/standalone.py` with the environment's interpreter, so it uses the standard +library and the installed contracts package only. MODULE_DEPENDENCY_UNDECLARED: the code imports +a distribution it does not declare (or one nothing installed provides); +MODULE_DEPENDENCY_UNUSED: it declares one it never imports; MODULE_MANIFEST_DEPENDENCIES_DIFFER: +`module.toml` and the distribution disagree; MODULE_MANIFEST_MISSING: no `module.toml`; +MODULE_TEST_IMPORTS_REPOSITORY: a test imports `tests`, `conformance` or `tools`; +MODULE_TEST_DEPENDENCY_UNDECLARED / _UNUSED: the tests' own dependencies against +`dependencies.tests`. +""" + +from __future__ import annotations + +import ast +import importlib +import importlib.metadata +import importlib.util +import re +import sys +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path + +REPOSITORY_HELPERS = frozenset({"tests", "conformance", "tools"}) +Owner = Callable[[str], "str | None"] + + +def canonical(distribution: str) -> str: + return re.sub(r"[-_.]+", "-", distribution).lower() + + +def imported_modules(source: str) -> set[str]: + """The absolute imports of one file, by their full dotted names.""" + found: set[str] = set() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + found.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.add(node.module) + return found + + +def _external(name: str) -> bool: + top = name.partition(".")[0] + return top not in sys.stdlib_module_names and top != "__future__" + + +def _used(own: str, imports: Iterable[str], owner_of: Owner) -> tuple[set[str], list[str]]: + used: set[str] = set() + unknown: list[str] = [] + for name in sorted(filter(_external, imports)): + owner = owner_of(name) + if owner is None: + unknown.append(name) + elif canonical(owner) != canonical(own): + used.add(canonical(owner)) + return used, unknown + + +def dependency_problems( + own: str, + imports: Iterable[str], + owner_of: Owner, + declared: Iterable[str], + manifest_runtime: Iterable[str], +) -> list[str]: + used, unknown = _used(own, imports, owner_of) + declared_names = {canonical(name) for name in declared} + problems = [ + f"MODULE_DEPENDENCY_UNDECLARED: {own} imports {name}, which nothing installed provides" + for name in unknown + ] + problems += [ + f"MODULE_DEPENDENCY_UNDECLARED: {own} imports {name} without declaring it" + for name in sorted(used - declared_names) + ] + problems += [ + f"MODULE_DEPENDENCY_UNUSED: {own} declares {name} and never imports it" + for name in sorted(declared_names - used) + ] + listed = {canonical(name) for name in manifest_runtime} + if listed != declared_names: + problems.append( + f"MODULE_MANIFEST_DEPENDENCIES_DIFFER: module.toml runtime {sorted(listed)}, " + f"distribution {sorted(declared_names)}" + ) + return problems + + +def suite_problems( + own: str, + imports_by_file: Mapping[str, set[str]], + owner_of: Owner, + runtime: Iterable[str], + tests: Iterable[str], +) -> list[str]: + problems: list[str] = [] + every: set[str] = set() + for path, imports in sorted(imports_by_file.items()): + helpers = sorted(n for n in imports if n.partition(".")[0] in REPOSITORY_HELPERS) + problems += [ + f"MODULE_TEST_IMPORTS_REPOSITORY: {path} imports {name}, a repository helper" + for name in helpers + ] + every |= imports - set(helpers) + used, unknown = _used(own, every, owner_of) + problems += [ + f"MODULE_TEST_DEPENDENCY_UNDECLARED: the tests import {name}; nothing installed has it" + for name in unknown + ] + runtime_names = {canonical(name) for name in runtime} + test_names = {canonical(name) for name in tests} + problems += [ + f"MODULE_TEST_DEPENDENCY_UNDECLARED: the tests import {name}; list it in dependencies.tests" + for name in sorted(used - runtime_names - test_names) + ] + problems += [ + f"MODULE_TEST_DEPENDENCY_UNUSED: dependencies.tests lists {name}, which no test imports" + for name in sorted(test_names - used) + ] + return problems + + +def file_owners() -> dict[Path, str]: + """Every installed file, with the distribution that installed it.""" + owners: dict[Path, str] = {} + for distribution in importlib.metadata.distributions(): + for file in distribution.files or (): + owners[Path(str(distribution.locate_file(file))).resolve()] = distribution.metadata[ + "Name" + ] + return owners + + +def owner_finder(owners: Mapping[Path, str]) -> Owner: + def owner_of(name: str) -> str | None: + try: + spec = importlib.util.find_spec(name) + except ImportError, ValueError: + return None + if spec is None: + return None + if spec.origin is not None and spec.has_location: + return owners.get(Path(spec.origin).resolve()) + for location in spec.submodule_search_locations or (): + inside = { + owner + for path, owner in owners.items() + if path.is_relative_to(Path(location).resolve()) + } + if len(inside) == 1: + return inside.pop() + return None + + return owner_of + + +def main(argv: list[str]) -> int: + namespace, name, tests = argv + manifest_module = importlib.import_module(f"{namespace}.contracts.manifest") + distribution = importlib.metadata.distribution(name) + paths = [Path(str(distribution.locate_file(f))) for f in distribution.files or ()] + manifests = [p for p in paths if p.name == "module.toml" and p.parent.parent.name == namespace] + if len(manifests) != 1: + print(f"MODULE_MANIFEST_MISSING: {name} ships {len(manifests)} module.toml, expected one") + return 1 + manifest = manifest_module.parse_manifest( + manifests[0].read_text("utf-8"), manifests[0].parent.name + ) + declared = [ + re.match(r"[A-Za-z0-9._-]+", requirement)[0] # type: ignore[index] + for requirement in distribution.requires or () + if "extra ==" not in requirement + ] + owner_of = owner_finder(file_owners()) + imports: set[str] = set() + for path in paths: + if path.suffix == ".py": + imports |= imported_modules(path.read_text("utf-8")) + problems = dependency_problems(name, imports, owner_of, declared, manifest.runtime) + by_file = { + path.relative_to(tests).as_posix(): imported_modules(path.read_text("utf-8")) + for path in sorted(Path(tests).rglob("*.py")) + } + problems += suite_problems(name, by_file, owner_of, manifest.runtime, manifest.tests) + for problem in problems: + print(problem) + print(f"module dependencies of {name}: {len(problems)} problem(s)") + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index b13846a9..6743329d 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -721,3 +721,36 @@ name = "semantic-corpus-clickhouse-26.8" run = "python -m conformance.runners.semantic_corpus --through clickhouse --switch-on" local = false reason = "needs a ClickHouse 26.8 LTS server (CORPUS_CLICKHOUSE), started by the job from its release archive" + +[[job]] +workflow = "interfaces.yml" +job = "interface-contracts" +kind = "check" +required = false +contexts = ["interface-contracts"] +digest = "db80658d940be1da" +setup = ["python -m pip install --quiet pip==26.3"] +[[job.check]] +# The record of the interface contracts is current (#486 G3). +name = "interface-record" +run = "python -m tools.contracts.interface_record --check" +local = true +[[job.check]] +# Within a major version an interface contract only grows since the last release (#486 G3). +name = "interface-compat" +run = "python -m tools.contracts.interface_compat" +local = true + +[[job]] +workflow = "interfaces.yml" +job = "standalone" +kind = "check" +required = false +contexts = ["standalone"] +digest = "bcb329418caea5d2" +setup = ["python -m pip install --quiet pip==26.3"] +[[job.check]] +# A module installs and passes its own tests alone, outside the repository (#486 G4). +name = "standalone-contracts" +run = "python -m tools.ci.standalone contracts" +local = true diff --git a/tools/ci/standalone.py b/tools/ci/standalone.py new file mode 100644 index 00000000..a13c344b --- /dev/null +++ b/tools/ci/standalone.py @@ -0,0 +1,155 @@ +"""The standalone job (#486 G4): a module built, installed and tested on its own, outside the +repository, with only the contracts distribution beside it. + + python -m tools.ci.standalone + +In a temporary directory outside the checkout: copy the module's package, build its sdist and, +from that sdist, its wheel (and the contracts wheel for any other module); create a clean +virtual environment; install the wheels and the module's test dependencies, every version +constrained to the lock file of this Python; run `tools/checks/module_dependencies.py` there; +and run the module's own tests with no repository configuration. A failed step, a failed test +or a skipped test (STANDALONE_TESTS_SKIPPED: a test that did not run never counts) fails the job. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tarfile +import tempfile +import tomllib +import xml.etree.ElementTree as ElementTree +from collections.abc import Sequence +from pathlib import Path + +from tools.ci.pins import pins +from tools.repository import MODULES, REPOSITORY_ROOT, identity + +PIP = "pip==26.3" +PYTEST_CONFIG = """[pytest] +addopts = --import-mode=importlib --strict-markers --strict-config -p no:cacheprovider +xfail_strict = true +filterwarnings = error +""" +# Runs one PEP 517 hook of setuptools (the build backend every package declares) in place. +BUILD = ( + "import sys; from setuptools import build_meta as b; " + "print(getattr(b, sys.argv[1])(sys.argv[2]))" +) + + +def outside_repository(directory: Path) -> Path: + if directory.resolve().is_relative_to(REPOSITORY_ROOT.resolve()): + raise SystemExit(f"STANDALONE_INSIDE_REPOSITORY: {directory} is inside the checkout") + return directory + + +def lock_file() -> Path: + name = "pylock.toml" if sys.version_info >= (3, 15) else "pylock.py314.toml" + return REPOSITORY_ROOT / name + + +def constraints(lock: Path) -> list[str]: + """`name==version` for every package of the lock file.""" + names = [package["name"] for package in tomllib.loads(lock.read_text("utf-8"))["packages"]] + return pins(lock, names) + + +def _step(name: str, command: Sequence[str | Path], cwd: Path) -> str: + print(f"standalone: {name}", flush=True) + done = subprocess.run([str(part) for part in command], cwd=cwd, capture_output=True, text=True) + if done.returncode != 0: + print(done.stdout[-4000:], done.stderr[-4000:], sep="\n") + raise SystemExit(f"STANDALONE_STEP_FAILED: {name} (exit {done.returncode})") + return done.stdout + + +def build(package: Path, work: Path) -> Path: + """The wheel of `package`, built from its sdist, itself built from a copy of the package.""" + source = work / "source" / package.name + shutil.copytree( + package, source, ignore=shutil.ignore_patterns("build", "*.egg-info", "__pycache__") + ) + dist = work / "dist" + dist.mkdir(exist_ok=True) + built = _step( + f"sdist of {package.name}", [sys.executable, "-c", BUILD, "build_sdist", dist], source + ) + sdist = dist / built.strip().splitlines()[-1] + unpacked = work / "unpacked" / package.name + with tarfile.open(sdist) as archive: + archive.extractall(unpacked, filter="data") + root = next(unpacked.iterdir()) + wheel = _step( + f"wheel of {package.name} from its sdist", + [sys.executable, "-c", BUILD, "build_wheel", dist], + root, + ) + return dist / wheel.strip().splitlines()[-1] + + +def run(module: str) -> int: + if module not in MODULES or module == "server": + raise SystemExit(f"STANDALONE_UNKNOWN_MODULE: {module}") + namespace = str(identity(REPOSITORY_ROOT)["python_namespace"]) + slug = str(identity(REPOSITORY_ROOT)["slug"]) + distribution = f"{slug}-{module}" + package = REPOSITORY_ROOT / "packages" / module + with tempfile.TemporaryDirectory(prefix="standalone-") as directory: + work = outside_repository(Path(directory)) + wheels = [build(package, work)] + if module != "contracts": + wheels.append(build(REPOSITORY_ROOT / "packages" / "contracts", work)) + environment = work / "environment" + _step("clean environment", [sys.executable, "-m", "venv", environment], work) + python = environment / "bin" / "python" + _step("pip", [python, "-m", "pip", "install", "--quiet", PIP], work) + pinned = work / "constraints.txt" + pinned.write_text("\n".join(constraints(lock_file())) + "\n", encoding="utf-8") + manifest = (package / "src" / namespace / module / "module.toml").read_text("utf-8") + tests = tomllib.loads(manifest)["dependencies"]["tests"] + _step( + f"install {distribution}" + + ("" if module == "contracts" else f" with only {slug}-contracts"), + [python, "-m", "pip", "install", "--quiet", "--constraint", pinned, *wheels, *tests], + work, + ) + suite = work / "suite" + shutil.copytree( + package / "tests", suite / "tests", ignore=shutil.ignore_patterns("__pycache__") + ) + (suite / "pytest.ini").write_text(PYTEST_CONFIG, encoding="utf-8") + checker = REPOSITORY_ROOT / "tools" / "checks" / "module_dependencies.py" + print( + _step( + "declared dependencies", + [python, "-I", checker, namespace, distribution, suite / "tests"], + work, + ) + ) + report = work / "junit.xml" + _step( + "own tests", + [python, "-I", "-m", "pytest", "-q", f"--junitxml={report}", "tests"], + suite, + ) + totals = ElementTree.parse(report).getroot().find("testsuite") + if totals is None: + raise SystemExit("STANDALONE_STEP_FAILED: no test report") + ran, skipped = int(totals.get("tests", "0")), int(totals.get("skipped", "0")) + if skipped or not ran: + raise SystemExit(f"STANDALONE_TESTS_SKIPPED: {skipped} of {ran} tests did not run") + print(f"standalone: {distribution} installs alone and passes its {ran} tests") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if len(arguments) != 1: + raise SystemExit("usage: python -m tools.ci.standalone ") + return run(arguments[0]) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/contracts/interface_compat.py b/tools/contracts/interface_compat.py new file mode 100644 index 00000000..ccf22111 --- /dev/null +++ b/tools/contracts/interface_compat.py @@ -0,0 +1,162 @@ +"""Interface-contract compatibility with the last release (#486 G3). + + python -m tools.contracts.interface_compat + +Compares the record of this checkout with the record committed at the last product release +(the newest `v` tag HEAD contains): within a major version a contract only grows. These +fail, unless the major version went up: a removed contract (INTERFACE_CONTRACT_REMOVED), a +lowered version (INTERFACE_VERSION_LOWERED), a removed operation (INTERFACE_OPERATION_REMOVED), +a changed signature or capability of an operation (INTERFACE_SIGNATURE_CHANGED), a removed +capability (INTERFACE_CAPABILITY_REMOVED), a removed or renamed error code +(INTERFACE_ERROR_CODE_REMOVED), a removed kit case (INTERFACE_KIT_CASE_REMOVED) or a changed +expectation (INTERFACE_KIT_EXPECTATION_CHANGED). Allowed: new contracts, operations, +capabilities, error codes, kit cases and new parameters with a default. No release tag fails +closed (INTERFACE_RELEASE_UNKNOWN); a release without a record released no interface contract. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +from orvex.contracts.interface import Version + +from tools.contracts.interface_record import RECORD, current_record +from tools.repository import REPOSITORY_ROOT + +# A product tag: `v` and a semantic version, its prerelease after a hyphen (`v0.3.1-alpha.1`). +PRODUCT_TAG = re.compile(r"v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[0-9A-Za-z.-]+)?") + + +def _operation_problems( + name: str, operation: str, old: dict[str, Any], new: dict[str, Any] +) -> list[str]: + where = f"{name}.{operation}" + problems = [] + if new["capability"] != old["capability"]: + problems.append( + f"INTERFACE_SIGNATURE_CHANGED: {where} moved from capability " + f"{old['capability']} to {new['capability']}" + ) + if new["returns"] != old["returns"]: + problems.append( + f"INTERFACE_SIGNATURE_CHANGED: {where} returns {new['returns']}, was {old['returns']}" + ) + before, after = old["parameters"], new["parameters"] + changed = ( + len(after) < len(before) + or [_shape(p) for p in after[: len(before)]] != [_shape(p) for p in before] + or any(p["default"] and not q["default"] for p, q in zip(before, after, strict=False)) + or any(not p["default"] for p in after[len(before) :]) + ) + if changed: + problems.append(f"INTERFACE_SIGNATURE_CHANGED: {where} changed its parameters") + return problems + + +def _shape(parameter: dict[str, Any]) -> tuple[str, str, str]: + return parameter["name"], parameter["kind"], parameter["type"] + + +def _contract_problems(name: str, old: dict[str, Any], new: dict[str, Any]) -> list[str]: + problems = [] + for operation, before in sorted(old["operations"].items()): + after = new["operations"].get(operation) + if after is None: + problems.append(f"INTERFACE_OPERATION_REMOVED: {name}.{operation}") + else: + problems += _operation_problems(name, operation, before, after) + problems += [ + f"INTERFACE_CAPABILITY_REMOVED: {name}: {capability}" + for capability in sorted(set(old["capabilities"]) - set(new["capabilities"])) + ] + problems += [ + f"INTERFACE_ERROR_CODE_REMOVED: {name}: {code}" + for code in sorted(set(old["errors"]) - set(new["errors"])) + ] + for case, before in sorted(old["kit"]["cases"].items()): + after = new["kit"]["cases"].get(case) + if after is None: + problems.append(f"INTERFACE_KIT_CASE_REMOVED: {name}: {case}") + elif after != before: + problems.append( + f"INTERFACE_KIT_EXPECTATION_CHANGED: {name}: {case}: {before} became {after}" + ) + return problems + + +def incompatibilities(released: dict[str, Any], current: dict[str, Any]) -> list[str]: + problems = [] + for name, old in sorted(released["contracts"].items()): + new = current["contracts"].get(name) + if new is None: + problems.append(f"INTERFACE_CONTRACT_REMOVED: {name}") + continue + before, after = Version.parse(old["version"]), Version.parse(new["version"]) + if after < before: + problems.append(f"INTERFACE_VERSION_LOWERED: {name}: {after}, released {before}") + elif after.major == before.major: + problems += _contract_problems(name, old, new) + return problems + + +def _git(root: Path, *arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *arguments], cwd=root, capture_output=True, text=True, check=False + ) + + +def last_release(root: Path) -> str: + """The newest product tag (`v`, docs/process/versioning.md) that HEAD contains.""" + found = _git( + root, + "for-each-ref", + "--merged", + "HEAD", + "--sort=-creatordate", + "--format=%(refname:short)", + "refs/tags/v*", + ) + tags = [tag for tag in found.stdout.split() if PRODUCT_TAG.fullmatch(tag)] + if found.returncode != 0 or not tags: + raise SystemExit( + "INTERFACE_RELEASE_UNKNOWN: no product release tag reaches HEAD (fetch the tags) " + + found.stderr.strip() + ) + return tags[0] + + +def released_record(root: Path, tag: str) -> dict[str, Any] | None: + path = RECORD.relative_to(REPOSITORY_ROOT).as_posix() + if _git(root, "cat-file", "-e", f"{tag}:{path}").returncode != 0: + return None + shown = _git(root, "show", f"{tag}:{path}") + if shown.returncode != 0: + raise SystemExit(f"INTERFACE_RELEASE_UNKNOWN: {tag}:{path}: {shown.stderr.strip()}") + record: dict[str, Any] = json.loads(shown.stdout) + return record + + +def run(root: Path) -> int: + tag = last_release(root) + released = released_record(root, tag) + if released is None: + print(f"interface contracts: release {tag} recorded none; nothing to compare") + return 0 + problems = incompatibilities(released, current_record()) + for problem in problems: + print(problem) + print(f"interface contracts: {len(problems)} incompatibility(ies) with release {tag}") + return 1 if problems else 0 + + +def main() -> int: + return run(REPOSITORY_ROOT) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/contracts/interface_record.py b/tools/contracts/interface_record.py new file mode 100644 index 00000000..982b7389 --- /dev/null +++ b/tools/contracts/interface_record.py @@ -0,0 +1,75 @@ +"""The record of the interface contracts (#486 G3): `packages/contracts/interface-record.json`, +each contract's description with its kit, as the compatibility check reads it at a release. + + python -m tools.contracts.interface_record # write the record + python -m tools.contracts.interface_record --check # INTERFACE_RECORD_STALE when it differs + +A contract without its kit (INTERFACE_KIT_MISSING) or a kit without its contract +(INTERFACE_KIT_ORPHANED) cannot be recorded. +""" + +from __future__ import annotations + +import argparse +import importlib +import json +import sys +from collections.abc import Mapping, Sequence +from typing import Any + +from orvex.contracts.interface import InterfaceContract +from orvex.contracts.kit import Kit +from orvex.contracts.registry import interface_contracts, kits + +from tools.repository import REPOSITORY_ROOT, identity + +RECORD = REPOSITORY_ROOT / "packages" / "contracts" / "interface-record.json" + + +def build_record( + contracts: Mapping[str, InterfaceContract], kits: Mapping[str, Kit] +) -> dict[str, Any]: + missing = sorted(set(contracts) - set(kits)) + if missing: + raise SystemExit(f"INTERFACE_KIT_MISSING: contracts {missing} ship no conformance kit") + orphaned = sorted(set(kits) - set(contracts)) + if orphaned: + raise SystemExit(f"INTERFACE_KIT_ORPHANED: kits {orphaned} have no registered contract") + return { + "contracts": { + name: {**contract.describe(), "kit": kits[name].describe()} + for name, contract in sorted(contracts.items()) + } + } + + +def render(record: Mapping[str, Any]) -> str: + return json.dumps(record, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + + +def current_record() -> dict[str, Any]: + """The record of the contracts this checkout declares.""" + package = importlib.import_module(f"{identity(REPOSITORY_ROOT)['python_namespace']}.contracts") + return build_record(interface_contracts(package), kits(package)) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true", help="fail when the record is stale") + arguments = parser.parse_args(argv) + text = render(current_record()) + if arguments.check: + if not RECORD.is_file() or RECORD.read_text("utf-8") != text: + print( + f"INTERFACE_RECORD_STALE: {RECORD.relative_to(RECORD.parents[2])}; run " + "python -m tools.contracts.interface_record" + ) + return 1 + print("interface record: current") + return 0 + RECORD.write_text(text, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 0d54bfa4385c89d388ef09e47aaa9faa391a0efc Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 00:35:41 -0400 Subject: [PATCH 03/21] test(contracts): the case matrix of the envelope, N and N-1, idempotency and adapters Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- packages/contracts/tests/sample_contract.py | 136 ++++--- packages/contracts/tests/test_adapter.py | 72 ++++ packages/contracts/tests/test_binding.py | 85 +++-- packages/contracts/tests/test_envelope.py | 146 ++++++++ packages/contracts/tests/test_interface.py | 92 +++-- packages/contracts/tests/test_kit.py | 372 ++++++++++++++++---- packages/contracts/tests/test_manifest.py | 2 +- packages/contracts/tests/test_registry.py | 52 ++- packages/server/tests/test_assembly.py | 16 +- tests/tools/test_interface_compat.py | 77 +++- tests/tools/test_standalone.py | 13 +- 11 files changed, 876 insertions(+), 187 deletions(-) create mode 100644 packages/contracts/tests/test_adapter.py create mode 100644 packages/contracts/tests/test_envelope.py diff --git a/packages/contracts/tests/sample_contract.py b/packages/contracts/tests/sample_contract.py index b0943eb7..b0abe487 100644 --- a/packages/contracts/tests/sample_contract.py +++ b/packages/contracts/tests/sample_contract.py @@ -1,4 +1,5 @@ -"""A small interface contract, an implementation and a kit for the tests of the format itself.""" +"""A small interface contract at version 1.1, its implementations at 1.1 and 1.0, and its kit, +for the tests of the format itself.""" from __future__ import annotations @@ -8,9 +9,11 @@ from typing import Protocol from zoneinfo import ZoneInfo -from orvex.contracts.binding import Bound -from orvex.contracts.interface import InterfaceContract, Operation, Version -from orvex.contracts.kit import Kit, KitCase +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation +from orvex.contracts.kit import Idempotent, Kit, KitCase +from orvex.contracts.kit_client import KitClient +from orvex.contracts.version import Version class LedgerError(Exception): @@ -31,44 +34,59 @@ class Reading: class Ledger(Protocol): - def put(self, reading: Reading) -> None: ... + def record(self, envelope: CallEnvelope, reading: Reading) -> int: ... - def get(self, name: str) -> Reading | None: ... + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: ... - def names(self) -> tuple[str, ...]: ... + def names(self, envelope: CallEnvelope) -> tuple[str, ...]: ... + + def count(self, envelope: CallEnvelope) -> int: ... LEDGER = InterfaceContract( name="ledger", - version=Version(1, 0), + version=Version(1, 1), protocol=Ledger, error_type=LedgerError, capabilities=("read", "write"), errors=("LEDGER_NAME_EMPTY",), operations=( - Operation("put", "write", ("LEDGER_NAME_EMPTY",)), - Operation("get", "read"), + Operation("record", "write", ("LEDGER_NAME_EMPTY",), changes_state=True), + Operation("readings", "read"), Operation("names", "read"), + Operation("count", "read", since=Version(1, 1)), ), ) -class MemoryLedger: - """A correct implementation of the ledger.""" +class MemoryLedgerV10: + """A correct implementation of the ledger at version 1.0, the released one: no `count`.""" def __init__(self) -> None: - self._readings: dict[str, Reading] = {} + self._readings: list[Reading] = [] + self._done: dict[str, int] = {} - def put(self, reading: Reading) -> None: + def record(self, envelope: CallEnvelope, reading: Reading) -> int: if not reading.name: raise LedgerError("LEDGER_NAME_EMPTY", "a reading needs a name", {"field": "name"}) - self._readings[reading.name] = reading + key = envelope.idempotency_key or "" + if key not in self._done: + self._readings.append(reading) + self._done[key] = len(self._readings) + return self._done[key] + + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: + return tuple(reading for reading in self._readings if reading.name == name) + + def names(self, envelope: CallEnvelope) -> tuple[str, ...]: + return tuple(sorted({reading.name for reading in self._readings})) + - def get(self, name: str) -> Reading | None: - return self._readings.get(name) +class MemoryLedger(MemoryLedgerV10): + """A correct implementation of the ledger at version 1.1.""" - def names(self) -> tuple[str, ...]: - return tuple(sorted(self._readings)) + def count(self, envelope: CallEnvelope) -> int: + return len(self._readings) READING = Reading( @@ -81,21 +99,21 @@ def _expect(condition: bool, text: str) -> None: raise AssertionError(text) -def round_trip(ledger: Bound) -> None: - ledger.put(READING) - found = ledger.get(READING.name) - _expect(found == READING, f"read back {found!r}") - _expect(isinstance(found, Reading) and str(found.amount) == "12.50", "the scale changed") - _expect(isinstance(found, Reading) and found.at.tzinfo == READING.at.tzinfo, "the zone changed") +def round_trip(ledger: KitClient) -> None: + ledger.record(READING) + found = ledger.readings(READING.name) + _expect(found == (READING,), f"read back {found!r}") + _expect(str(found[0].amount) == "12.50", "the scale changed") + _expect(found[0].at.tzinfo == READING.at.tzinfo, "the zone changed") -def missing(ledger: Bound) -> None: - _expect(ledger.get("absent") is None, "a missing name is not None") +def missing(ledger: KitClient) -> None: + _expect(ledger.readings("absent") == (), "an unknown name has readings") -def empty_name_refused(ledger: Bound) -> None: +def empty_name_refused(ledger: KitClient) -> None: try: - ledger.put(Reading("", Decimal(1), READING.at)) + ledger.record(Reading("", Decimal(1), READING.at)) except LedgerError as error: _expect(error.code == "LEDGER_NAME_EMPTY", f"code {error.code}") _expect(error.details == {"field": "name"}, f"details {error.details}") @@ -103,29 +121,67 @@ def empty_name_refused(ledger: Bound) -> None: raise AssertionError("an empty name was accepted") -def names_sorted(ledger: Bound) -> None: - for name in ("b", "a"): - ledger.put(Reading(name, Decimal(0), READING.at)) - _expect(ledger.names() == ("a", "b"), "names are not sorted") +def names_sorted(ledger: KitClient) -> None: + for name in ("b", "a", "b"): + ledger.record(Reading(name, Decimal(0), READING.at)) + _expect(ledger.names() == ("a", "b"), "names are not sorted and distinct") + + +def counted(ledger: KitClient) -> None: + for name in ("a", "a"): + ledger.record(Reading(name, Decimal(0), READING.at)) + _expect(ledger.count() == 2, "the count is not the number of readings") + + +def record_once(ledger: KitClient, envelope: CallEnvelope) -> object: + return ledger.send("record", envelope, READING) + + +def observe_readings(ledger: KitClient) -> object: + return ledger.readings(READING.name) LEDGER_KIT = Kit( contract=LEDGER, - recorded=4, + recorded=6, cases=( KitCase( - "put.round-trip-exact", - "put", + "record.round-trip-exact", + "record", "a reading read back keeps its name, amount scale and time zone", round_trip, ), - KitCase("get.missing-is-none", "get", "an unknown name reads as None", missing), + KitCase("readings.missing-is-empty", "readings", "an unknown name has no reading", missing), KitCase( - "put.empty-name-refused", - "put", + "record.empty-name-refused", + "record", "an empty name is refused with LEDGER_NAME_EMPTY", empty_name_refused, ), - KitCase("names.sorted", "names", "names come back sorted", names_sorted), + KitCase("names.sorted", "names", "names come back sorted and distinct", names_sorted), + KitCase( + "record.idempotent", + "record", + "a reading recorded twice with one idempotency key is kept once, with one number", + Idempotent(record_once, observe_readings), + ), + KitCase( + "count.readings", + "count", + "the count is the number of readings recorded", + counted, + since=Version(1, 1), + ), ), ) + +# A complete envelope for direct calls in the tests (a fresh key where an effect is counted). +ENVELOPE = CallEnvelope( + contract_name="ledger", + contract_version="1.1", + correlation_id="correlation-1", + actor="actor-1", + project_id="project-1", + revision="revision-1", + idempotency_key="key-1", +) diff --git a/packages/contracts/tests/test_adapter.py b/packages/contracts/tests/test_adapter.py new file mode 100644 index 00000000..1009c438 --- /dev/null +++ b/packages/contracts/tests/test_adapter.py @@ -0,0 +1,72 @@ +"""Public test adapters (#486 G4): a module runs its own operations with only the contracts it +requires, each supplied by an adapter, and without the modules that only consume it.""" + +from __future__ import annotations + +from decimal import Decimal + +import pytest +from orvex.contracts.adapter import ContractAdapter, required_adapters +from orvex.contracts.binding import Bound +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.kit_run import verify_kit +from orvex.contracts.manifest import Manifest, Required +from orvex.contracts.version import Version + +from .sample_contract import ENVELOPE, LEDGER, LEDGER_KIT, READING, MemoryLedger, Reading + +LEDGER_ADAPTER = ContractAdapter(LEDGER, MemoryLedger) + + +def needing(*required: Required) -> Manifest: + return Manifest("billing", (), required, (), ()) + + +class Billing: + """A sample module that requires the ledger contract and nothing else.""" + + def __init__(self, ledger: Bound) -> None: + self._ledger = ledger + + def bill(self, name: str, amount: Decimal) -> int: + number: int = self._ledger.record(ENVELOPE, Reading(name, amount, READING.at)) + return number + + +def test_a_module_runs_its_own_operations_with_only_its_adapters() -> None: + manifest = needing(Required("ledger", Version(1, 0), ("write",))) + given = required_adapters(manifest, {"ledger": LEDGER_ADAPTER}) + assert list(given) == ["ledger"] + ledger = Bound(LEDGER, given["ledger"].make(), serialized=True) + assert Billing(ledger).bill("meter-1", Decimal("3.10")) == 1 + + +def test_the_adapter_is_a_full_implementation_that_passes_its_kit() -> None: + assert verify_kit(LEDGER_KIT, LEDGER_ADAPTER.make) + + +def test_a_module_without_requirements_needs_no_adapter() -> None: + assert required_adapters(needing(), {}) == {} + + +@pytest.mark.parametrize( + "found", [{}, {"clock": ContractAdapter(LEDGER, MemoryLedger)}], ids=["none", "another"] +) +def test_a_required_contract_without_an_adapter_is_refused( + found: dict[str, ContractAdapter], +) -> None: + manifest = needing(Required("ledger", Version(1, 0), ("read",))) + with pytest.raises(InterfaceContractError) as raised: + required_adapters(manifest, found) + assert raised.value.code == "MODULE_ADAPTER_MISSING" + assert "billing requires ledger 1.0: no public test adapter" in raised.value.message + + +@pytest.mark.parametrize("version", [Version(1, 2), Version(2, 0)]) +def test_an_adapter_at_an_incompatible_version_is_refused(version: Version) -> None: + with pytest.raises(InterfaceContractError) as raised: + required_adapters( + needing(Required("ledger", version, ("read",))), {"ledger": LEDGER_ADAPTER} + ) + assert raised.value.code == "MODULE_ADAPTER_MISSING" + assert "the adapter is at 1.1" in raised.value.message diff --git a/packages/contracts/tests/test_binding.py b/packages/contracts/tests/test_binding.py index 759d8c88..02a71d6f 100644 --- a/packages/contracts/tests/test_binding.py +++ b/packages/contracts/tests/test_binding.py @@ -12,11 +12,20 @@ import pytest from orvex.contracts.binding import Bound from orvex.contracts.codec import Codec +from orvex.contracts.envelope import CallEnvelope from orvex.contracts.interface_errors import InterfaceContractError from orvex.contracts.kit_run import run_kit, verify_kit from pydantic import BaseModel -from .sample_contract import LEDGER, LEDGER_KIT, READING, LedgerError, MemoryLedger, Reading +from .sample_contract import ( + ENVELOPE, + LEDGER, + LEDGER_KIT, + READING, + LedgerError, + MemoryLedger, + Reading, +) class Color(Enum): @@ -37,7 +46,7 @@ class ReadingChild(Reading): pass -CODEC = Codec(frozenset({Reading, Color, Shape})) +CODEC = Codec(frozenset({Reading, Color, Shape, CallEnvelope})) VALUES: list[object] = [ None, @@ -70,6 +79,7 @@ class ReadingChild(Reading): Color.RED, Shape(side=Decimal("2.50")), READING, + ENVELOPE, (), [], {}, @@ -151,13 +161,30 @@ class LiveNames(tuple[str, ...]): class LiveLedger(MemoryLedger): """A planted violation: `names` hands out a live object instead of a value.""" - def names(self) -> tuple[str, ...]: - return LiveNames(sorted(self._readings)) + def names(self, envelope: CallEnvelope) -> tuple[str, ...]: + return LiveNames(super().names(envelope)) class WrongCode(MemoryLedger): - def get(self, name: str) -> Reading | None: - raise LedgerError("LEDGER_GONE", "not declared for get", {}) + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: + raise LedgerError("LEDGER_GONE", "not declared for readings", {}) + + +class Crashing(MemoryLedger): + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: + raise KeyError(name) + + +class Seen(MemoryLedger): + """Keeps every envelope it is called with.""" + + def __init__(self) -> None: + super().__init__() + self.envelopes: list[CallEnvelope] = [] + + def names(self, envelope: CallEnvelope) -> tuple[str, ...]: + self.envelopes.append(envelope) + return super().names(envelope) class TestBound: @@ -173,44 +200,57 @@ def test_only_the_contract_s_operations_are_reachable(self, serialized: bool) -> @pytest.mark.parametrize("serialized", [False, True]) def test_an_implementation_without_the_operation_is_refused(self, serialized: bool) -> None: class Partial: - def get(self, name: str) -> None: - return None + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: + return () bound = Bound(LEDGER, Partial(), serialized=serialized) with pytest.raises(InterfaceContractError) as raised: - bound.names() + bound.names(ENVELOPE) assert raised.value.code == "INTERFACE_OPERATION_MISSING" + @pytest.mark.parametrize("serialized", [False, True]) + def test_the_implementation_receives_the_envelope(self, serialized: bool) -> None: + ledger = Seen() + Bound(LEDGER, ledger, serialized=serialized).names(ENVELOPE) + assert ledger.envelopes == [ENVELOPE] + assert (ledger.envelopes[0] is ENVELOPE) is not serialized + def test_serialized_calls_cross_as_copies(self) -> None: ledger = MemoryLedger() - Bound(LEDGER, ledger, serialized=True).put(READING) - kept = ledger.get(READING.name) - assert kept == READING - assert kept is not READING - assert Bound(LEDGER, ledger, serialized=True).get(READING.name) is not kept + Bound(LEDGER, ledger, serialized=True).record(ENVELOPE, READING) + kept = ledger.readings(ENVELOPE, READING.name) + assert kept == (READING,) + assert kept[0] is not READING + again = Bound(LEDGER, ledger, serialized=True).readings(ENVELOPE, READING.name) + assert again[0] is not kept[0] in_process = MemoryLedger() - Bound(LEDGER, in_process, serialized=False).put(READING) - assert in_process.get(READING.name) is READING + Bound(LEDGER, in_process, serialized=False).record(ENVELOPE, READING) + assert in_process.readings(ENVELOPE, READING.name)[0] is READING def test_a_declared_error_crosses_with_its_code_message_and_details(self) -> None: + bound = Bound(LEDGER, MemoryLedger(), serialized=True) with pytest.raises(LedgerError) as raised: - Bound(LEDGER, MemoryLedger(), serialized=True).put(Reading("", Decimal(1), READING.at)) + bound.record(ENVELOPE, Reading("", Decimal(1), READING.at)) assert raised.value.code == "LEDGER_NAME_EMPTY" assert raised.value.message == "a reading needs a name" assert raised.value.details == {"field": "name"} assert raised.value.__cause__ is None @pytest.mark.parametrize("serialized", [False, True]) - def test_an_undeclared_error_code_is_refused_in_both_bindings(self, serialized: bool) -> None: + @pytest.mark.parametrize("implementation", [WrongCode, Crashing]) + def test_an_undeclared_error_is_refused_in_both_bindings( + self, serialized: bool, implementation: type[MemoryLedger] + ) -> None: with pytest.raises(InterfaceContractError) as raised: - Bound(LEDGER, WrongCode(), serialized=serialized).get("a") + Bound(LEDGER, implementation(), serialized=serialized).readings(ENVELOPE, "a") assert raised.value.code == "INTERFACE_ERROR_UNDECLARED" + assert (raised.value.__cause__ is None) is serialized def test_a_live_result_is_refused_when_serialized(self) -> None: ledger = LiveLedger() - assert Bound(LEDGER, ledger, serialized=False).names() == () + assert Bound(LEDGER, ledger, serialized=False).names(ENVELOPE) == () with pytest.raises(InterfaceContractError) as raised: - Bound(LEDGER, ledger, serialized=True).names() + Bound(LEDGER, ledger, serialized=True).names(ENVELOPE) assert raised.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" @@ -221,8 +261,9 @@ def test_the_kit_fails_the_serialized_run_and_the_runs_differ(self) -> None: serialized = run_kit(LEDGER_KIT, LiveLedger, serialized=True) failed = [result.id for result in serialized.results if not result.passed] assert failed == ["names.sorted"] + assert run_kit(LEDGER_KIT, LiveLedger, serialized=False).results[3].passed with pytest.raises(InterfaceContractError) as raised: verify_kit(LEDGER_KIT, LiveLedger) assert raised.value.code == "KIT_FAILED" assert "KIT_BINDINGS_DIFFER" in raised.value.message - assert "names.sorted (serialized)" in raised.value.message + assert "names.sorted (serialized, consumer 1.1, provider 1.1)" in raised.value.message diff --git a/packages/contracts/tests/test_envelope.py b/packages/contracts/tests/test_envelope.py new file mode 100644 index 00000000..644de3cd --- /dev/null +++ b/packages/contracts/tests/test_envelope.py @@ -0,0 +1,146 @@ +"""The call envelope (#486 §2 item 6, G3): every binding refuses a call missing a required field, +or at a version the provider cannot serve, before the implementation runs.""" + +from __future__ import annotations + +import dataclasses +from typing import Any + +import pytest +from orvex.contracts.binding import Bound +from orvex.contracts.envelope import REQUIRED, CallEnvelope +from orvex.contracts.interface_errors import ContractCallRefused +from orvex.contracts.version import Version + +from .sample_contract import ENVELOPE, LEDGER, READING, MemoryLedger, MemoryLedgerV10, Reading + +BINDINGS = pytest.mark.parametrize("serialized", [False, True], ids=["in-process", "serialized"]) + + +class Untouchable: + """An implementation that fails the test if any operation of it is reached.""" + + def __getattr__(self, name: str) -> object: + raise AssertionError(f"the implementation was reached: {name}") + + +def refused( + envelope: object, + operation: str = "names", + provided: Version | None = None, + *, + serialized: bool, +) -> ContractCallRefused: + bound = Bound(LEDGER, Untouchable(), serialized=serialized, provided=provided) + arguments: tuple[object, ...] = (READING,) if operation == "record" else () + with pytest.raises(ContractCallRefused) as raised: + bound.call(operation, envelope, *arguments) + return raised.value + + +@BINDINGS +@pytest.mark.parametrize("field", REQUIRED) +@pytest.mark.parametrize("value", [None, "", " ", 7], ids=["absent", "empty", "blank", "int"]) +def test_a_call_missing_a_required_field_is_refused( + serialized: bool, field: str, value: object +) -> None: + planted: dict[str, Any] = {field: value} + envelope = dataclasses.replace(ENVELOPE, **planted) + error = refused(envelope, serialized=serialized) + assert error.code == "CONTRACT_ENVELOPE_INVALID" + assert error.details == {"contract": "ledger", "operation": "names", "field": field} + + +@BINDINGS +@pytest.mark.parametrize("key", [None, "", " "]) +def test_an_operation_that_changes_state_needs_an_idempotency_key( + serialized: bool, key: str | None +) -> None: + envelope = dataclasses.replace(ENVELOPE, idempotency_key=key) + error = refused(envelope, "record", serialized=serialized) + assert error.code == "CONTRACT_ENVELOPE_INVALID" + assert error.details["field"] == "idempotency_key" + names = Bound(LEDGER, MemoryLedger(), serialized=serialized).names(envelope) + assert names == () + + +@BINDINGS +@pytest.mark.parametrize( + "envelope", + [None, "ledger", {"contract_name": "ledger"}, dataclasses.astuple(ENVELOPE)], + ids=["none", "text", "dict", "tuple"], +) +def test_anything_but_an_envelope_is_refused(serialized: bool, envelope: object) -> None: + error = refused(envelope, serialized=serialized) + assert error.code == "CONTRACT_ENVELOPE_INVALID" + assert error.details["field"] == "envelope" + + +@BINDINGS +def test_an_envelope_for_another_contract_is_refused(serialized: bool) -> None: + error = refused(dataclasses.replace(ENVELOPE, contract_name="store"), serialized=serialized) + assert error.code == "CONTRACT_ENVELOPE_INVALID" + assert error.details["field"] == "contract_name" + + +@BINDINGS +@pytest.mark.parametrize("version", ["1", "1.0.0", "v1.1", "01.1"]) +def test_a_version_that_is_not_major_minor_is_refused(serialized: bool, version: str) -> None: + error = refused(dataclasses.replace(ENVELOPE, contract_version=version), serialized=serialized) + assert error.code == "CONTRACT_ENVELOPE_INVALID" + assert error.details["field"] == "contract_version" + + +@BINDINGS +@pytest.mark.parametrize("version", ["2.0", "2.1", "3.4"]) +def test_an_incompatible_major_version_is_refused_explicitly( + serialized: bool, version: str +) -> None: + error = refused(dataclasses.replace(ENVELOPE, contract_version=version), serialized=serialized) + assert error.code == "CONTRACT_VERSION_UNSUPPORTED" + assert error.details["requested"] == version + assert error.details["provided"] == "1.1" + + +@BINDINGS +def test_an_operation_newer_than_the_provider_is_refused(serialized: bool) -> None: + error = refused(ENVELOPE, "count", Version(1, 0), serialized=serialized) + assert error.code == "CONTRACT_VERSION_UNSUPPORTED" + assert "count exists from 1.1; the provider serves 1.0" in error.message + + +@BINDINGS +def test_an_operation_newer_than_the_caller_s_version_is_refused(serialized: bool) -> None: + older = dataclasses.replace(ENVELOPE, contract_version="1.0") + error = refused(older, "count", serialized=serialized) + assert error.code == "CONTRACT_VERSION_UNSUPPORTED" + assert "the caller speaks 1.0" in error.message + + +@BINDINGS +def test_a_newer_caller_is_served_by_an_older_provider_for_what_it_has(serialized: bool) -> None: + bound = Bound(LEDGER, MemoryLedgerV10(), serialized=serialized, provided=Version(1, 0)) + assert bound.record(ENVELOPE, READING) == 1 + assert bound.readings(ENVELOPE, READING.name) == (READING,) + + +@BINDINGS +def test_an_older_caller_is_served_by_a_newer_provider(serialized: bool) -> None: + older = dataclasses.replace(ENVELOPE, contract_version="1.0") + bound = Bound(LEDGER, MemoryLedger(), serialized=serialized) + assert bound.record(older, Reading("a", READING.amount, READING.at)) == 1 + assert bound.names(older) == ("a",) + + +@pytest.mark.parametrize("provided", [Version(1, 2), Version(2, 0)]) +def test_a_binding_never_serves_a_version_the_contract_does_not_describe( + provided: Version, +) -> None: + with pytest.raises(ContractCallRefused) as raised: + Bound(LEDGER, MemoryLedger(), serialized=False, provided=provided) + assert raised.value.code == "CONTRACT_VERSION_UNSUPPORTED" + + +def test_the_envelope_holds_the_required_fields_and_the_key() -> None: + assert dataclasses.fields(CallEnvelope)[-1].name == "idempotency_key" + assert [f.name for f in dataclasses.fields(CallEnvelope)][:-1] == list(REQUIRED) diff --git a/packages/contracts/tests/test_interface.py b/packages/contracts/tests/test_interface.py index e3cf4a70..683f481c 100644 --- a/packages/contracts/tests/test_interface.py +++ b/packages/contracts/tests/test_interface.py @@ -12,8 +12,10 @@ from typing import Any, Literal, Protocol import pytest -from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.version import Version from pydantic import BaseModel from .sample_contract import LEDGER, LedgerError, Reading @@ -85,40 +87,56 @@ def test_a_version_provides_its_major_at_a_lower_or_equal_minor(self) -> None: assert not Version(2, 0).provides(Version(1, 0)) assert not Version(1, 0).provides(Version(2, 0)) + def test_the_previous_version_is_the_previous_minor_of_the_major(self) -> None: + assert Version(1, 3).previous() == Version(1, 2) + assert Version(2, 0).previous() is None + class TestDescription: def test_the_description_names_every_part_of_the_contract(self) -> None: described = LEDGER.describe() assert described["name"] == "ledger" - assert described["version"] == "1.0" + assert described["version"] == "1.1" assert described["capabilities"] == ["read", "write"] assert described["errors"] == ["LEDGER_NAME_EMPTY"] operations = described["operations"] assert isinstance(operations, dict) - assert sorted(operations) == ["get", "names", "put"] - assert operations["put"] == { + assert sorted(operations) == ["count", "names", "readings", "record"] + envelope = { + "name": "envelope", + "kind": "positional-or-keyword", + "type": f"{CallEnvelope.__module__}.CallEnvelope", + "default": False, + } + assert operations["record"] == { "capability": "write", + "changes_state": True, + "since": "1.0", "errors": ["LEDGER_NAME_EMPTY"], "parameters": [ + envelope, { "name": "reading", "kind": "positional-or-keyword", "type": READING_TYPE, "default": False, - } + }, ], - "returns": "None", + "returns": "int", } - assert operations["get"]["returns"] == f"{READING_TYPE} | None" - assert operations["names"]["returns"] == "tuple[str, ...]" + assert operations["readings"]["returns"] == f"tuple[{READING_TYPE}, ...]" + assert operations["names"]["changes_state"] is False + assert operations["count"]["since"] == "1.1" def test_operations_are_found_by_name_and_by_capability(self) -> None: - assert LEDGER.operation("get").capability == "read" - assert [o.name for o in LEDGER.operations_of("read")] == ["get", "names"] + assert LEDGER.operation("readings").capability == "read" + assert [o.name for o in LEDGER.operations_of("read")] == ["readings", "names", "count"] + assert LEDGER.since(LEDGER.operation("count")) == Version(1, 1) + assert LEDGER.since(LEDGER.operation("names")) == Version(1, 0) assert code_of(lambda: LEDGER.operation("drop")) == "INTERFACE_OPERATION_UNKNOWN" -def _plain(self: object, value: int) -> None: ... +def _plain(self: object, envelope: CallEnvelope, value: int) -> None: ... class TestRefusals: @@ -140,6 +158,8 @@ def test_a_valid_probe_is_accepted(self) -> None: {"operations": (Operation("use", "use"), Operation("other", "use"))}, {"operations": ()}, {"error_type": int}, + {"operations": (Operation("use", "use", since=Version(1, 1)),)}, + {"operations": (Operation("use", "use", since=Version(2, 0)),)}, ], ids=lambda changes: ",".join(changes), ) @@ -163,12 +183,12 @@ def test_unsupported_operation_can_never_be_declared(self, where: str) -> None: ) def test_unannotated_and_variadic_parameters_are_refused(self) -> None: - def unannotated(self: object, value): # type: ignore[no-untyped-def] # noqa: ANN001, ANN202 + def unannotated(self: object, envelope: CallEnvelope, value): # type: ignore[no-untyped-def] # noqa: ANN001, ANN202 return value - def variadic(self: object, *values: int) -> None: ... + def variadic(self: object, envelope: CallEnvelope, *values: int) -> None: ... - def keywords(self: object, **values: int) -> None: ... + def keywords(self: object, envelope: CallEnvelope, **values: int) -> None: ... for method in (unannotated, variadic, keywords): with pytest.raises(InterfaceContractError) as raised: @@ -177,19 +197,44 @@ def keywords(self: object, **values: int) -> None: ... def _takes(annotation: object) -> Callable[..., object]: - def method(self: object, value: Any) -> None: ... # noqa: ANN401 - replaced below + def method(self: object, envelope: CallEnvelope, value: Any) -> None: ... # noqa: ANN401 - method.__annotations__ = {"value": annotation, "return": type(None)} + method.__annotations__ = {"envelope": CallEnvelope, "value": annotation, "return": type(None)} return method def _returns(annotation: object) -> Callable[..., object]: - def method(self: object) -> None: ... + def method(self: object, envelope: CallEnvelope) -> None: ... - method.__annotations__ = {"return": annotation} + method.__annotations__ = {"envelope": CallEnvelope, "return": annotation} return method +def _no_envelope(self: object, value: int) -> None: ... + + +def _envelope_second(self: object, value: int, envelope: CallEnvelope) -> None: ... + + +def _envelope_keyword(self: object, *, envelope: CallEnvelope) -> None: ... + + +def _envelope_untyped(self: object, envelope: str) -> None: ... + + +class TestEnvelopeParameter: + @pytest.mark.parametrize( + "method", [_no_envelope, _envelope_second, _envelope_keyword, _envelope_untyped] + ) + def test_an_operation_without_the_envelope_first_is_refused( + self, method: Callable[..., object] + ) -> None: + with pytest.raises(InterfaceContractError) as raised: + contract_for(method) + assert raised.value.code == "INTERFACE_CONTRACT_INVALID" + assert "the first parameter is `envelope: CallEnvelope`" in raised.value.message + + SERIALIZABLE: list[object] = [ int, bool, @@ -247,9 +292,14 @@ def test_a_live_object_may_never_cross(self, annotation: object) -> None: ) def test_the_classes_that_cross_are_collected(self) -> None: - assert contract_for(_takes(Outer)).value_classes() == frozenset({Outer, Inner}) - assert contract_for(_returns(list[Color | Shape])).value_classes() == {Color, Shape} - assert LEDGER.value_classes() == {Reading} + found = contract_for(_takes(Outer)).value_classes() + assert found == frozenset({CallEnvelope, Outer, Inner}) + assert contract_for(_returns(list[Color | Shape])).value_classes() == { + CallEnvelope, + Color, + Shape, + } + assert LEDGER.value_classes() == {CallEnvelope, Reading} def test_replace_validates_again(self) -> None: assert code_of(lambda: dataclasses.replace(LEDGER, name="Bad")) == ( diff --git a/packages/contracts/tests/test_kit.py b/packages/contracts/tests/test_kit.py index 94ee5c09..78da7604 100644 --- a/packages/contracts/tests/test_kit.py +++ b/packages/contracts/tests/test_kit.py @@ -1,17 +1,32 @@ -"""The conformance-kit format and its run (#486 G1): every case, a recorded count, no skips.""" +"""The conformance-kit format and its run (#486 G1, G3): every case, a recorded count, no skips, +idempotency, the envelope cases, and N against N-1 in both directions.""" from __future__ import annotations import dataclasses from collections.abc import Callable +from typing import Any import pytest -from orvex.contracts.binding import Bound -from orvex.contracts.interface_errors import InterfaceContractError -from orvex.contracts.kit import Kit, KitCase -from orvex.contracts.kit_run import kit_problems, run_kit, verify_kit - -from .sample_contract import LEDGER, LEDGER_KIT, MemoryLedger, Reading +from orvex.contracts.envelope import CallEnvelope, envelope_refusal +from orvex.contracts.interface import InterfaceContract, Operation +from orvex.contracts.interface_errors import ContractCallRefused, InterfaceContractError +from orvex.contracts.kit import Idempotent, Kit, KitCase +from orvex.contracts.kit_client import KitClient +from orvex.contracts.kit_run import consumers, kit_problems, run_kit, verify_kit +from orvex.contracts.version import Version + +from .sample_contract import ( + LEDGER, + LEDGER_KIT, + MemoryLedger, + MemoryLedgerV10, + Reading, + observe_readings, + record_once, +) + +V10, V11 = Version(1, 0), Version(1, 1) def code_of(build: Callable[[], object]) -> str: @@ -20,87 +35,258 @@ def code_of(build: Callable[[], object]) -> str: return raised.value.code -class RoundingLedger(MemoryLedger): - """A planted implementation that fails one case: it drops the amount's scale.""" - - def put(self, reading: Reading) -> None: - super().put(dataclasses.replace(reading, amount=reading.amount.normalize())) - - -def test_a_correct_implementation_passes_every_case_in_both_bindings() -> None: - in_process, serialized = verify_kit(LEDGER_KIT, MemoryLedger) - assert not in_process.serialized - assert serialized.serialized - for report in (in_process, serialized): - assert report.contract == "ledger" - assert [r.id for r in report.results] == [c.id for c in LEDGER_KIT.cases] - assert all(r.passed for r in report.results) - - -def test_a_planted_implementation_failing_one_case_fails_the_kit() -> None: - report = run_kit(LEDGER_KIT, RoundingLedger, serialized=False) - assert [r.id for r in report.results if not r.passed] == ["put.round-trip-exact"] +def failure(make: Callable[[], object], provided: Version | None = None) -> str: with pytest.raises(InterfaceContractError) as raised: - verify_kit(LEDGER_KIT, RoundingLedger) + verify_kit(LEDGER_KIT, make, provided) assert raised.value.code == "KIT_FAILED" - assert "KIT_CASE_FAILED: put.round-trip-exact (in process)" in raised.value.message - assert "KIT_CASE_FAILED: put.round-trip-exact (serialized)" in raised.value.message - - -def test_each_case_gets_a_fresh_implementation() -> None: - made: list[MemoryLedger] = [] - - def make() -> MemoryLedger: - made.append(MemoryLedger()) - return made[-1] - - run_kit(LEDGER_KIT, make, serialized=False) - assert len(made) == LEDGER_KIT.recorded + return raised.value.message -def test_a_run_with_one_case_fewer_fails() -> None: - report = run_kit(LEDGER_KIT, MemoryLedger, serialized=False) - shorter = dataclasses.replace(report, results=report.results[:-1]) - problems = kit_problems(LEDGER_KIT, shorter) - assert problems == ["KIT_RUN_INCOMPLETE: names.sorted did not run (in process)"] - - -def test_a_skip_is_a_failure_since_the_kit_has_no_skips() -> None: - def skipping(ledger: Bound) -> None: - pytest.skip("not today") +class RoundingLedger(MemoryLedger): + """A planted implementation that fails one case: it drops the amount's scale.""" - cases = (dataclasses.replace(LEDGER_KIT.cases[0], check=skipping), *LEDGER_KIT.cases[1:]) - kit = dataclasses.replace(LEDGER_KIT, cases=cases) - report = run_kit(kit, MemoryLedger, serialized=False) - assert [r.passed for r in report.results] == [False, True, True, True] - assert "not today" in report.results[0].detail + def record(self, envelope: CallEnvelope, reading: Reading) -> int: + normalized = dataclasses.replace(reading, amount=reading.amount.normalize()) + return super().record(envelope, normalized) + + +class ForgetfulLedger(MemoryLedger): + """A planted violation: it changes state twice for one idempotency key.""" + + def record(self, envelope: CallEnvelope, reading: Reading) -> int: + unique = dataclasses.replace(envelope, idempotency_key=f"{len(self._readings)}") + return super().record(unique, reading) + + +class InertLedger(MemoryLedger): + """An implementation whose `record` has no effect: its idempotency case proves nothing.""" + + def record(self, envelope: CallEnvelope, reading: Reading) -> int: + return 0 + + +class NewOnlyLedger(MemoryLedger): + """A planted violation: version 1.1 broke its 1.0 consumers, whose names come back empty.""" + + def names(self, envelope: CallEnvelope) -> tuple[str, ...]: + return super().names(envelope) if envelope.contract_version == "1.1" else () + + +class WrongCountV10(MemoryLedgerV10): + """A planted older provider that answers the newer operation wrongly instead of refusing.""" + + def count(self, envelope: CallEnvelope) -> int: + return -1 + + +class TestRun: + def test_a_correct_implementation_passes_every_case_both_ways_by_n_and_n_1(self) -> None: + reports = verify_kit(LEDGER_KIT, MemoryLedger) + assert [(r.consumer, r.serialized) for r in reports] == [ + (V10, False), + (V10, True), + (V11, False), + (V11, True), + ] + for report in reports: + assert report.contract == "ledger" + assert report.provided == V11 + expected = [ + c.id for c in LEDGER_KIT.cases if report.consumer.provides(LEDGER_KIT.since(c)) + ] + assert [r.id for r in report.results] == expected + assert all(r.passed for r in (*report.results, *report.envelope_results)) + assert len(reports[0].results) == 5 + assert len(reports[2].results) == 6 + + def test_a_planted_implementation_failing_one_case_fails_the_kit(self) -> None: + report = run_kit(LEDGER_KIT, RoundingLedger, serialized=False) + assert [r.id for r in report.results if not r.passed] == ["record.round-trip-exact"] + message = failure(RoundingLedger) + for binding_name in ("in process", "serialized"): + assert ( + f"KIT_CASE_FAILED: record.round-trip-exact ({binding_name}, consumer 1.1, " + "provider 1.1)" in message + ) + + def test_each_case_gets_a_fresh_implementation(self) -> None: + made: list[MemoryLedger] = [] + + def make() -> MemoryLedger: + made.append(MemoryLedger()) + return made[-1] + + report = run_kit(LEDGER_KIT, make, serialized=False) + assert len(made) == LEDGER_KIT.recorded + len(report.envelope_results) + + def test_a_run_with_one_case_fewer_fails(self) -> None: + report = run_kit(LEDGER_KIT, MemoryLedger, serialized=False) + shorter = dataclasses.replace(report, results=report.results[:-1]) + assert kit_problems(LEDGER_KIT, shorter) == [ + "KIT_RUN_INCOMPLETE: count.readings did not run (in process, consumer 1.1, " + "provider 1.1)" + ] + fewer = dataclasses.replace(report, envelope_results=report.envelope_results[:-1]) + assert kit_problems(LEDGER_KIT, fewer)[0].startswith("KIT_RUN_INCOMPLETE") + + def test_a_skip_is_a_failure_since_the_kit_has_no_skips(self) -> None: + def skipping(ledger: KitClient) -> None: + pytest.skip("not today") + + cases = (dataclasses.replace(LEDGER_KIT.cases[0], check=skipping), *LEDGER_KIT.cases[1:]) + kit = dataclasses.replace(LEDGER_KIT, cases=cases) + report = run_kit(kit, MemoryLedger, serialized=False) + assert [r.passed for r in report.results] == [False, True, True, True, True, True] + assert "not today" in report.results[0].detail + + +class TestIdempotency: + def test_an_operation_that_changes_state_twice_for_one_key_fails(self) -> None: + report = run_kit(LEDGER_KIT, ForgetfulLedger, serialized=False) + failed = [r for r in report.results if not r.passed] + assert [r.id for r in failed] == ["record.idempotent"] + assert "KIT_IDEMPOTENCY_BROKEN" in failed[0].detail + assert "KIT_CASE_FAILED: record.idempotent" in failure(ForgetfulLedger) + + def test_a_call_without_effect_proves_nothing_and_fails(self) -> None: + report = run_kit(LEDGER_KIT, InertLedger, serialized=True) + idempotent = next(r for r in report.results if r.id == "record.idempotent") + assert "KIT_IDEMPOTENCY_NO_EFFECT" in idempotent.detail + + def test_each_call_gets_a_fresh_key_and_a_repeat_sends_the_same_envelope(self) -> None: + sent: list[CallEnvelope] = [] + + class Spy(MemoryLedger): + def record(self, envelope: CallEnvelope, reading: Reading) -> int: + sent.append(envelope) + return super().record(envelope, reading) + + run_kit(LEDGER_KIT, Spy, serialized=False) + # round-trip 0, empty name 1, names.sorted 2-4, idempotent 5-6, count 7-8 + assert len(sent) == 9 + assert sent[5] == sent[6] + assert len({envelope.idempotency_key for envelope in sent[2:5]}) == 3 + assert all(e.correlation_id.startswith("kit-") and e.actor for e in sent) + + +class TestEnvelopeCases: + def test_every_operation_gets_its_envelope_cases(self) -> None: + report = run_kit(LEDGER_KIT, MemoryLedger, serialized=False) + ids = {r.id for r in report.envelope_results} + assert "envelope.record.idempotency_key-missing" in ids + assert "envelope.names.idempotency_key-missing" not in ids + assert "envelope.count.correlation_id-blank" in ids + assert "envelope.readings.major-2" in ids + assert len(ids) == 4 * 14 + 1 + older = run_kit(LEDGER_KIT, MemoryLedger, serialized=False, consumer=V10) + assert not any(r.id.startswith("envelope.count.") for r in older.envelope_results) + + def test_a_binding_that_accepts_an_incompatible_major_silently_fails( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + def lax(*arguments: Any) -> ContractCallRefused | None: # noqa: ANN401 + refusal = envelope_refusal(*arguments) + if refusal is not None and refusal.details.get("requested") == "2.0": + return None + return refusal + + monkeypatch.setattr("orvex.contracts.binding.envelope_refusal", lax) + message = failure(MemoryLedger) + assert "KIT_ENVELOPE_CASE_FAILED: envelope.names.major-2" in message + assert "expected CONTRACT_VERSION_UNSUPPORTED" in message + + def test_a_binding_that_accepts_a_call_missing_a_field_fails( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + def lax(*arguments: Any) -> ContractCallRefused | None: # noqa: ANN401 + refusal = envelope_refusal(*arguments) + if refusal is not None and refusal.details.get("field") == "correlation_id": + return None + return refusal + + monkeypatch.setattr("orvex.contracts.binding.envelope_refusal", lax) + message = failure(MemoryLedger) + assert "KIT_ENVELOPE_CASE_FAILED: envelope.names.correlation_id-missing" in message + assert "reached ['names']" in message + + +class TestVersions: + def test_consumers_are_n_1_n_and_the_newest(self) -> None: + assert consumers(LEDGER, V11) == (V10, V11) + assert consumers(LEDGER, V10) == (V10, V11) + grown = dataclasses.replace(LEDGER, version=Version(1, 2)) + assert consumers(grown, V10) == (V10, Version(1, 2)) + assert consumers(grown, V11) == (V10, V11, Version(1, 2)) + + def test_a_provider_at_n_1_passes_the_kit_of_n(self) -> None: + reports = verify_kit(LEDGER_KIT, MemoryLedgerV10, provided=V10) + newest = [r for r in reports if r.consumer == V11] + for report in newest: + outcome = {r.id: r.outcome for r in report.results} + assert outcome.pop("count.readings") == "refused" + assert set(outcome.values()) == {"passed"} + + def test_a_consumer_at_n_1_broken_by_an_additive_change_fails(self) -> None: + message = failure(NewOnlyLedger) + assert "KIT_CASE_FAILED: names.sorted (in process, consumer 1.0, provider 1.1)" in message + assert "consumer 1.1" not in message + + def test_a_provider_at_n_1_that_answers_a_newer_operation_is_still_refused(self) -> None: + reports = verify_kit(LEDGER_KIT, WrongCountV10, provided=V10) + newest = next(r for r in reports if r.consumer == V11) + assert next(r for r in newest.results if r.id == "count.readings").outcome == "refused" + + def test_a_provider_at_n_1_with_a_wrong_answer_fails(self) -> None: + def wrong(ledger: KitClient) -> None: + if ledger.names() != ("z",): + raise AssertionError("not z") + + case = KitCase("names.z", "names", "a newer guarantee on an old operation", wrong, V11) + kit = Kit(LEDGER, recorded=7, cases=(*LEDGER_KIT.cases, case)) + with pytest.raises(InterfaceContractError) as raised: + verify_kit(kit, MemoryLedgerV10, provided=V10) + message = raised.value.message + assert "KIT_CASE_FAILED: names.z (in process, consumer 1.1, provider 1.0)" in message + + def test_a_version_the_contract_does_not_describe_cannot_run(self) -> None: + newer = Version(2, 0) + assert code_of( + lambda: run_kit(LEDGER_KIT, MemoryLedger, serialized=False, consumer=newer) + ) == ("KIT_INVALID") + assert code_of(lambda: verify_kit(LEDGER_KIT, MemoryLedger, Version(1, 2))) == ( + "CONTRACT_VERSION_UNSUPPORTED" + ) def test_the_description_lists_every_case_and_the_recorded_count() -> None: - assert LEDGER_KIT.describe() == { - "recorded": 4, - "cases": { - case.id: {"operation": case.operation, "expectation": case.expectation} - for case in LEDGER_KIT.cases - }, + described = LEDGER_KIT.describe() + assert described["recorded"] == 6 + cases = described["cases"] + assert isinstance(cases, dict) + assert list(cases) == [case.id for case in LEDGER_KIT.cases] + assert cases["record.idempotent"] == { + "operation": "record", + "expectation": LEDGER_KIT.cases[4].expectation, + "idempotency": True, + "since": "1.0", } + assert cases["count.readings"]["since"] == "1.1" + assert cases["names.sorted"]["idempotency"] is False -def _noop(ledger: Bound) -> None: - ledger.get("x") +def _noop(ledger: KitClient) -> None: + ledger.names() -CASE = KitCase("get.extra", "get", "an extra case", _noop) +CASE = KitCase("names.extra", "names", "an extra case", _noop) class TestDeclarationRefusals: def test_a_recorded_count_that_differs_from_the_cases_is_refused(self) -> None: - assert code_of(lambda: dataclasses.replace(LEDGER_KIT, recorded=3)) == ( - "KIT_CASE_COUNT_CHANGED" - ) - assert code_of(lambda: dataclasses.replace(LEDGER_KIT, recorded=5)) == ( - "KIT_CASE_COUNT_CHANGED" - ) + for recorded in (5, 7): + with pytest.raises(InterfaceContractError) as raised: + dataclasses.replace(LEDGER_KIT, recorded=recorded) + assert raised.value.code == "KIT_CASE_COUNT_CHANGED" def test_an_operation_without_a_case_is_refused(self) -> None: cases = tuple(case for case in LEDGER_KIT.cases if case.operation != "names") @@ -108,15 +294,36 @@ def test_an_operation_without_a_case_is_refused(self) -> None: "KIT_OPERATION_UNCOVERED" ) + def test_an_operation_that_changes_state_without_an_idempotency_case_is_refused( + self, + ) -> None: + cases = tuple(case for case in LEDGER_KIT.cases if case.id != "record.idempotent") + assert code_of(lambda: Kit(LEDGER, recorded=len(cases), cases=cases)) == ( + "KIT_IDEMPOTENCY_UNCOVERED" + ) + @pytest.mark.parametrize( "case", [ - dataclasses.replace(CASE, id="put.round-trip-exact"), + dataclasses.replace(CASE, id="record.round-trip-exact"), dataclasses.replace(CASE, id="Bad Id"), dataclasses.replace(CASE, operation="drop"), dataclasses.replace(CASE, expectation=" "), + dataclasses.replace(CASE, since=Version(1, 2)), + dataclasses.replace(CASE, since=Version(2, 0)), + dataclasses.replace(CASE, operation="count", since=V10), + dataclasses.replace(CASE, check=Idempotent(record_once, observe_readings)), + ], + ids=[ + "duplicate", + "id", + "operation", + "expectation", + "since-newer", + "since-major", + "before-operation", + "idempotency-on-read", ], - ids=["duplicate", "id", "operation", "expectation"], ) def test_an_invalid_case_is_refused(self, case: KitCase) -> None: cases = (*LEDGER_KIT.cases, case) @@ -124,4 +331,21 @@ def test_an_invalid_case_is_refused(self, case: KitCase) -> None: def test_an_extra_valid_case_is_accepted(self) -> None: cases = (*LEDGER_KIT.cases, CASE) - assert Kit(LEDGER, recorded=5, cases=cases).recorded == 5 + assert Kit(LEDGER, recorded=7, cases=cases).recorded == 7 + + +def test_a_contract_of_one_read_needs_no_idempotency_case() -> None: + read_only = InterfaceContract( + name="names", + version=V10, + protocol=type( + "Names", + (), + {"names": LEDGER.protocol.names}, # type: ignore[attr-defined] + ), + error_type=LEDGER.error_type, + capabilities=("read",), + errors=(), + operations=(Operation("names", "read"),), + ) + assert Kit(read_only, recorded=1, cases=(CASE,)).recorded == 1 diff --git a/packages/contracts/tests/test_manifest.py b/packages/contracts/tests/test_manifest.py index 6910a34a..5d986256 100644 --- a/packages/contracts/tests/test_manifest.py +++ b/packages/contracts/tests/test_manifest.py @@ -6,9 +6,9 @@ import orvex.contracts import pytest -from orvex.contracts.interface import Version from orvex.contracts.interface_errors import InterfaceContractError from orvex.contracts.manifest import Provided, Required, parse_manifest +from orvex.contracts.version import Version FULL = """ module = "store" diff --git a/packages/contracts/tests/test_registry.py b/packages/contracts/tests/test_registry.py index 6534ce90..0ebcd2da 100644 --- a/packages/contracts/tests/test_registry.py +++ b/packages/contracts/tests/test_registry.py @@ -1,4 +1,5 @@ -"""Finding the interface contracts and kits a contracts package ships, by their place.""" +"""Finding the interface contracts, kits and public test adapters a contracts package ships, by +their place; every shipped adapter passes its contract's whole kit (#486 G4).""" from __future__ import annotations @@ -10,18 +11,32 @@ import orvex.contracts import pytest from orvex.contracts.interface_errors import InterfaceContractError -from orvex.contracts.registry import interface_contracts, kits +from orvex.contracts.kit_run import verify_kit +from orvex.contracts.registry import adapters, interface_contracts, kits + +ADAPTER = """ +from orvex.contracts.adapter import ContractAdapter +from .interface import CLOCK + +class FixedClock: + def now(self, envelope) -> int: + return {answer} + +CLOCK_ADAPTER = ContractAdapter(CLOCK, FixedClock) +""" CONTRACT = """ from typing import Protocol -from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation +from orvex.contracts.version import Version class Error(Exception): def __init__(self, code: str, message: str, details: dict[str, str]) -> None: super().__init__(code) class Clock(Protocol): - def now(self) -> int: ... + def now(self, envelope: CallEnvelope) -> int: ... {name} = InterfaceContract( name="{contract}", version=Version(1, 0), protocol=Clock, error_type=Error, @@ -34,7 +49,8 @@ def now(self) -> int: ... from .interface import CLOCK def ticks(clock) -> None: - clock.now() + if clock.now() != 7: + raise AssertionError("the clock does not tick at 7") CLOCK_KIT = Kit(CLOCK, recorded=1, cases=(KitCase("now.ticks", "now", "it answers", ticks),)) """ @@ -69,6 +85,7 @@ def test_contracts_and_kits_are_found_in_their_places(package: Path) -> None: "clock/__init__.py": "", "clock/interface.py": CONTRACT.format(name="CLOCK", contract="clock"), "clock/kit.py": KIT, + "clock/adapter.py": ADAPTER.format(answer=7), "plain/__init__.py": "", "plain/v1.py": "X = 1\n", }, @@ -77,6 +94,24 @@ def test_contracts_and_kits_are_found_in_their_places(package: Path) -> None: assert list(interface_contracts(loaded)) == ["clock"] assert list(kits(loaded)) == ["clock"] assert kits(loaded)["clock"].contract is interface_contracts(loaded)["clock"] + assert list(adapters(loaded)) == ["clock"] + assert verify_kit(kits(loaded)["clock"], adapters(loaded)["clock"].make) + + +def test_a_planted_adapter_that_fails_its_kit_is_caught(package: Path) -> None: + write( + package, + { + "clock/__init__.py": "", + "clock/interface.py": CONTRACT.format(name="CLOCK", contract="clock"), + "clock/kit.py": KIT, + "clock/adapter.py": ADAPTER.format(answer=8), + }, + ) + loaded = importlib.import_module(package.name) + with pytest.raises(InterfaceContractError) as raised: + verify_kit(kits(loaded)["clock"], adapters(loaded)["clock"].make) + assert raised.value.code == "KIT_FAILED" def test_two_contracts_with_one_name_are_refused(package: Path) -> None: @@ -96,3 +131,10 @@ def test_two_contracts_with_one_name_are_refused(package: Path) -> None: def test_every_contract_the_distribution_ships_has_its_kit() -> None: assert set(kits(orvex.contracts)) == set(interface_contracts(orvex.contracts)) + + +def test_every_adapter_the_distribution_ships_passes_its_contract_s_whole_kit() -> None: + shipped = kits(orvex.contracts) + for name, adapter in adapters(orvex.contracts).items(): + assert adapter.contract is shipped[name].contract + verify_kit(shipped[name], adapter.make) diff --git a/packages/server/tests/test_assembly.py b/packages/server/tests/test_assembly.py index 5eb3e154..0208ad8f 100644 --- a/packages/server/tests/test_assembly.py +++ b/packages/server/tests/test_assembly.py @@ -9,8 +9,10 @@ from typing import Protocol import pytest -from orvex.contracts.interface import InterfaceContract, Operation, Version +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation from orvex.contracts.manifest import Manifest, Provided, Required +from orvex.contracts.version import Version from orvex.server import cli from orvex.server.assembly import ( AssemblyRefused, @@ -29,11 +31,11 @@ def __init__(self, code: str, message: str, details: dict[str, str]) -> None: class Cache(Protocol): - def get(self, key: str) -> bytes | None: ... + def get(self, envelope: CallEnvelope, key: str) -> bytes | None: ... - def put(self, key: str, value: bytes) -> None: ... + def put(self, envelope: CallEnvelope, key: str, value: bytes) -> None: ... - def size(self) -> int: ... + def size(self, envelope: CallEnvelope) -> int: ... CACHE = InterfaceContract( @@ -43,7 +45,11 @@ def size(self) -> int: ... error_type=CacheError, capabilities=("read", "write"), errors=(), - operations=(Operation("get", "read"), Operation("size", "read"), Operation("put", "write")), + operations=( + Operation("get", "read"), + Operation("size", "read"), + Operation("put", "write", changes_state=True), + ), ) diff --git a/tests/tools/test_interface_compat.py b/tests/tools/test_interface_compat.py index 45600e9d..5c4ecf23 100644 --- a/tests/tools/test_interface_compat.py +++ b/tests/tools/test_interface_compat.py @@ -10,10 +10,12 @@ import orvex.contracts import pytest -from orvex.contracts.binding import Bound -from orvex.contracts.interface import InterfaceContract, Operation, Version -from orvex.contracts.kit import Kit, KitCase +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation +from orvex.contracts.kit import Idempotent, Kit, KitCase +from orvex.contracts.kit_client import KitClient from orvex.contracts.registry import interface_contracts, kits +from orvex.contracts.version import Version from tools.contracts import interface_compat, interface_record from tools.contracts.interface_compat import incompatibilities, last_release, released_record @@ -27,15 +29,19 @@ def __init__(self, code: str, message: str, details: dict[str, str]) -> None: class Meter(Protocol): - def read(self, meter: str) -> int: ... + def read(self, envelope: CallEnvelope, meter: str) -> int: ... - def reset(self, meter: str) -> None: ... + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... -def check(meter: Bound) -> None: +def check(meter: KitClient) -> None: meter.read("m") +def sent(meter: KitClient, envelope: CallEnvelope) -> object: + return meter.send("reset", envelope, "m") + + def contract(**changes: Any) -> InterfaceContract: # noqa: ANN401 fields: dict[str, Any] = { "name": "meter", @@ -73,33 +79,33 @@ def codes(found: list[str]) -> list[str]: class ReadOnly(Protocol): - def read(self, meter: str) -> int: ... + def read(self, envelope: CallEnvelope, meter: str) -> int: ... class Renamed(Protocol): - def read(self, name: str) -> int: ... + def read(self, envelope: CallEnvelope, name: str) -> int: ... - def reset(self, meter: str) -> None: ... + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... class Retyped(Protocol): - def read(self, meter: str) -> str: ... + def read(self, envelope: CallEnvelope, meter: str) -> str: ... - def reset(self, meter: str) -> None: ... + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... class MoreRequired(Protocol): - def read(self, meter: str, at: int) -> int: ... + def read(self, envelope: CallEnvelope, meter: str, at: int) -> int: ... - def reset(self, meter: str) -> None: ... + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... class MoreOptional(Protocol): - def read(self, meter: str, at: int | None = None) -> int: ... + def read(self, envelope: CallEnvelope, meter: str, at: int | None = None) -> int: ... - def reset(self, meter: str) -> None: ... + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... - def total(self) -> int: ... + def total(self, envelope: CallEnvelope) -> int: ... def removed_operation() -> dict[str, Any]: @@ -192,17 +198,52 @@ def test_appending_is_compatible(self) -> None: operations=( Operation("read", "read", ("METER_UNKNOWN",)), Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED")), - Operation("total", "sum", ("METER_OVERFLOW",)), + Operation("total", "sum", ("METER_OVERFLOW",), since=Version(1, 1)), ), ) cases = ( KitCase("read.known", "read", "a known meter reads its count", check), KitCase("reset.zero", "reset", "a reset meter reads zero", check), - KitCase("total.sums", "total", "the total sums every meter", check), + KitCase("total.sums", "total", "the total sums every meter", check, Version(1, 1)), + KitCase("read.after-reset", "read", "a read after a reset", check, Version(1, 1)), ) assert incompatibilities(RELEASED, recorded(grown, kit(grown, *cases))) == [] assert incompatibilities({"contracts": {}}, RELEASED) == [] + def test_an_addition_claiming_the_released_version_fails(self) -> None: + grown = contract( + version=Version(1, 1), + protocol=MoreOptional, + operations=( + Operation("read", "read", ("METER_UNKNOWN",)), + Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED")), + Operation("total", "read"), + ), + ) + cases = ( + KitCase("read.known", "read", "a known meter reads its count", check), + KitCase("reset.zero", "reset", "a reset meter reads zero", check), + KitCase("total.sums", "total", "the total sums every meter", check), + ) + found = incompatibilities(RELEASED, recorded(grown, kit(grown, *cases))) + assert codes(found) == ["INTERFACE_ADDITION_UNVERSIONED"] * 2 + assert "meter.total is new since 1.0 and claims 1.0" in found[0] + + def test_an_operation_that_starts_changing_state_fails(self) -> None: + changed = contract( + operations=( + Operation("read", "read", ("METER_UNKNOWN",)), + Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED"), changes_state=True), + ) + ) + cases = ( + KitCase("read.known", "read", "a known meter reads its count", check), + KitCase("reset.zero", "reset", "a reset meter reads zero", Idempotent(sent, check)), + ) + found = incompatibilities(RELEASED, recorded(changed, kit(changed, *cases))) + assert "INTERFACE_SIGNATURE_CHANGED" in codes(found) + assert any("reset changes_state is True, was False" in problem for problem in found) + @pytest.mark.parametrize("build", [removed_operation, renamed_error, changed_expectation]) def test_a_new_major_version_may_break(self, build: Any) -> None: # noqa: ANN401 broken = build() diff --git a/tests/tools/test_standalone.py b/tests/tools/test_standalone.py index 2c9f2b01..4f76b488 100644 --- a/tests/tools/test_standalone.py +++ b/tests/tools/test_standalone.py @@ -6,14 +6,17 @@ from pathlib import Path import pytest +from orvex.contracts.manifest import Manifest, Required +from orvex.contracts.version import Version from tools.checks.module_dependencies import ( + adapter_problems, dependency_problems, imported_modules, suite_problems, ) from tools.ci.standalone import constraints, outside_repository -from tools.repository import REPOSITORY_ROOT +from tools.repository import REPOSITORY_ROOT, identity OWNERS = { "orvex.store": "orvex-store", @@ -128,3 +131,11 @@ def test_the_environment_is_constrained_to_the_lock() -> None: pinned = constraints(REPOSITORY_ROOT / "pylock.toml") assert "pydantic==2.14.3" in pinned assert all("==" in line for line in pinned) + + +def test_a_required_contract_without_a_public_test_adapter_fails() -> None: + needs = Manifest("billing", (), (Required("ledger", Version(1, 0), ("read",)),), (), ()) + namespace = str(identity(REPOSITORY_ROOT)["python_namespace"]) + found = adapter_problems(needs, namespace) + assert [problem.split(":", 1)[0] for problem in found] == ["MODULE_ADAPTER_MISSING"] + assert adapter_problems(Manifest("billing", (), (), (), ()), namespace) == [] From 4fdb62ac8d3058b2d25ed5ae03f75ea77c45b68a Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 00:35:41 -0400 Subject: [PATCH 04/21] feat(contracts): call envelope, N and N-1 kit runs, idempotency and test adapters Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- ...-interface-contracts-kits-and-manifests.md | 18 +- docs/process/git-workflow.md | 4 + packages/contracts/README.md | 2 +- .../contracts/src/orvex/contracts/adapter.py | 52 ++++ .../contracts/src/orvex/contracts/binding.py | 84 ++++-- .../contracts/src/orvex/contracts/envelope.py | 103 +++++++ .../src/orvex/contracts/interface.py | 59 ++-- .../src/orvex/contracts/interface_errors.py | 15 +- packages/contracts/src/orvex/contracts/kit.py | 63 ++++- .../src/orvex/contracts/kit_client.py | 65 +++++ .../contracts/src/orvex/contracts/kit_run.py | 255 +++++++++++++++--- .../contracts/src/orvex/contracts/manifest.py | 3 +- .../contracts/src/orvex/contracts/registry.py | 16 +- .../contracts/src/orvex/contracts/version.py | 38 +++ tools/checks/module_dependencies.py | 27 +- tools/ci/checks.toml | 4 +- tools/contracts/interface_compat.py | 44 ++- tools/repo/main-ruleset.json | 8 + 18 files changed, 733 insertions(+), 127 deletions(-) create mode 100644 packages/contracts/src/orvex/contracts/adapter.py create mode 100644 packages/contracts/src/orvex/contracts/envelope.py create mode 100644 packages/contracts/src/orvex/contracts/kit_client.py create mode 100644 packages/contracts/src/orvex/contracts/version.py diff --git a/docs/adr/0033-interface-contracts-kits-and-manifests.md b/docs/adr/0033-interface-contracts-kits-and-manifests.md index 86b55658..a98db4a1 100644 --- a/docs/adr/0033-interface-contracts-kits-and-manifests.md +++ b/docs/adr/0033-interface-contracts-kits-and-manifests.md @@ -1,22 +1,24 @@ # 0033 — Interface contracts, conformance kits and module manifests (amends 0004 rules 2–4) -**Status:** Proposed (2026-11-14, #486) +**Status:** Proposed (2026-11-15, #486 revision 7) ## Context ADR-0004 makes modules talk only through contracts, but its contracts are data contracts: they say what crosses a boundary, never which operations a module offers. On 2026-11-14 no module imported another module's internals, yet `server`, the conformance runners, `tools/` and the repository tests used 65 internal paths of the modules (#486 §1), the boundaries check exempted `server` as the composition root, and no module had ever been installed or tested outside the repository. Nothing proved that another implementation of a module could replace it. ## Decision -1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version, groups the operations into capabilities and lists the error codes each may raise through the contract's error type. Only values cross: the annotated inputs and outputs are plain values, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`; its kit in `orvex.contracts..kit` (`orvex.contracts.registry`). -2. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise and their expectation in words, a recorded case count and no skip list. A kit leaves no operation without a case. It runs on a fresh implementation per case, once in process and once through the serializing binding (`orvex.contracts.kit_run`, `binding`, `codec`), and passes only when every case passes both ways with the same results; a skip is a failure. An implementation that passes its contract's whole kit may replace the module. -3. **Consumers know contracts, not modules** (amends rule 3). `server` stays the only composition root, but only `server/assembly.py` names implementations, and only through each module's public `api` package. Every other consumer (`server`, the conformance runners, `tools/`, the repository tests) imports only `orvex.contracts`. `tools/checks/consumers.py` enforces it, in report mode until #486 step 7 makes it blocking and removes the `server` exemption. -4. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` attribute that implements each) and requires (version, capabilities), and the distributions its code and its tests import. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation or more than one, a version that does not provide the required one, a missing capability, or an implementation lacking an operation of what it claims. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. -5. **Compatibility of behavior.** The record of every interface contract and its kit (`packages/contracts/interface-record.json`, `tools/contracts/interface_record.py`) is compared with the record at the last product release (`tools/contracts/interface_compat.py`): within a major version operations, signatures, capabilities, error codes and kit cases with their expectations may only be added. A break needs a new major version (rule 6 of ADR-0004 applies to it). -6. **Standalone modules.** `tools/ci/standalone.py` builds a module's sdist and, from it, its wheel, installs it in a clean environment outside the repository with only `orvex-contracts` beside it, checks that its imports equal its declared dependencies and that its tests import no repository helper, and runs its own tests there, where a skipped test fails. +1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version (`orvex.contracts.version`), groups the operations into capabilities, lists the error codes each may raise through the contract's error type, marks the operations that change state and the version each exists from. Only values cross: the annotated inputs and outputs are plain values, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`, its kit in `orvex.contracts..kit` and its public test adapter, when it has one, in `orvex.contracts..adapter` (`orvex.contracts.registry`). +2. **The call envelope.** Every operation takes a `CallEnvelope` (`orvex.contracts.envelope`) as its first argument: `contract_name` and `contract_version` (the contract's version, never the module's or a physical format's), `correlation_id`, the effective `actor` or a verifiable reference to it, `project_id` and `revision`, and an `idempotency_key`, required on every operation that changes state. Every binding (`orvex.contracts.binding`) validates it before the implementation runs and fails closed with a typed `ContractCallRefused`: CONTRACT_ENVELOPE_INVALID for a missing, blank or mistyped field, a missing key or another contract's envelope; CONTRACT_VERSION_UNSUPPORTED for another major version or an operation that the provider's or the caller's version does not have. A caller at a newer minor version is served for the operations the provider has. +3. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise, their expectation in words and the version they exist from, a recorded case count and no skip list. A kit leaves no operation without a case, and each operation that changes state has an idempotency case: the call is sent twice with one key and its effect must happen exactly once. The runner (`orvex.contracts.kit_run`, `kit_client`, `codec`) runs every case on a fresh implementation, once in process and once through the serializing binding, by a consumer at N-1 and at N, and adds the envelope cases the contract implies (each missing field and key, another contract, another major version), which must be refused before the implementation is reached. A provider at N-1 runs the kit of N: the cases of its version pass, the newer ones pass or are refused with CONTRACT_VERSION_UNSUPPORTED, never answered wrongly. An implementation that passes its contract's whole kit may replace the module. +4. **Consumers know contracts, not modules** (amends rule 3). `server` stays the only composition root, but only `server/assembly.py` names implementations, and only through each module's public `api` package. Every other consumer (`server`, the conformance runners, `tools/`, the repository tests) imports only `orvex.contracts`. `tools/checks/consumers.py` enforces it, in report mode until #486 step 7 makes it blocking and removes the `server` exemption. +5. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` attribute that implements each) and requires (version, capabilities), and the distributions its code and its tests import. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation or more than one, a version that does not provide the required one, a missing capability, or an implementation lacking an operation of what it claims. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. +6. **Compatibility of behavior.** The record of every interface contract and its kit (`packages/contracts/interface-record.json`, `tools/contracts/interface_record.py`) is compared with the record at the last product release (`tools/contracts/interface_compat.py`): within a major version operations, signatures, capabilities, error codes and kit cases with their expectations may only be added, each addition existing only from a version after the release. A break needs a new major version (rule 6 of ADR-0004 applies to it). Rule 3 runs N against N-1 in both directions: each module's own tests run its kit by consumers at N-1 and N, and an implementation at N-1 runs the current kit with its own version as `provided`. The job that builds a module's last released implementation for that run arrives with the first released interface contract (#486 step 2); until then no released implementation exists. +7. **Standalone modules.** `tools/ci/standalone.py` builds a module's sdist and, from it, its wheel, installs it in a clean environment outside the repository with only `orvex-contracts` beside it, checks that its imports equal its declared dependencies, that its tests import no repository helper and that every contract it requires has a public test adapter (`orvex.contracts.adapter`, itself passing the contract's whole kit), and runs its own tests there, where a skipped test fails. A module thus starts and runs its own operations without the modules that only consume it. ## Alternatives considered +- **An envelope carried beside the call (context variables or a session object) instead of as the first argument:** less to write per operation, but invisible in the signature, impossible to serialize as a value and easy to lose across a process boundary. - **Interface contracts as language-neutral documents with generated protocols:** symmetrical with the data contracts, but it adds a generator and a second description of each signature to keep in step; the record of rule 5 already gives a language-neutral view of each interface for comparison. - **A second, complete implementation of every module** (#486 revision 3): proves substitution but duplicates about 41,000 lines; the kits, the assembly check and one real substitution (the semantic result cache, #486 step 5) prove it at a fraction of the cost. -- **Process isolation now:** belongs to the workers of #156; rule 1's value-only crossing and the serializing binding make the contracts ready for it. +- **Process isolation now:** belongs to the workers of #156; rule 1's value-only crossing, rule 2's envelope and the serializing binding make the contracts ready for it. ## Consequences Every module gains a contract, a kit, a manifest and an `api` package, one module per step of #486, and its consumers move onto them; module error classes used across modules move into `orvex.contracts`. CI gains the interface record and compatibility check and the standalone job. The documents that say `server` knows the modules (`architecture/overview.md`, `AGENTS.md` §5, `engineering.md` §2 and §8, `modules/contracts.md`) are corrected when this ADR is accepted. diff --git a/docs/process/git-workflow.md b/docs/process/git-workflow.md index aaca7196..26aa0e9e 100644 --- a/docs/process/git-workflow.md +++ b/docs/process/git-workflow.md @@ -124,6 +124,10 @@ packages/contracts/src/orvex/contracts/ generated contract types !packages/contracts/src/orvex/contracts/module.toml hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/registry.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/serializable.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/adapter.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/envelope.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/kit_client.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/version.py hand-written: counted (#486, interface contracts) tests/contracts/generated/ generated contract fixtures !tests/contracts/generated/__init__.py hand-written: counted web/src/lib/contracts/ generated TypeScript types diff --git a/packages/contracts/README.md b/packages/contracts/README.md index 21bce09c..f3c7f70c 100644 --- a/packages/contracts/README.md +++ b/packages/contracts/README.md @@ -3,7 +3,7 @@ Every module may import this package. It holds: - Python types generated from [`contracts/`](../../contracts/README.md) by `python -m tools.contracts.generate` (`/v.py`), never edited by hand; -- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, version), `kit.py` and `kit_run.py` (conformance kits: cases, recorded count, no skips, run in process and serialized), `binding.py`, `codec.py` and `serializable.py` (what may cross a contract), `manifest.py` (`module.toml`), `registry.py` and `interface_errors.py`; +- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, state changes) and `version.py`; `envelope.py` (the call envelope every binding validates); `kit.py`, `kit_client.py` and `kit_run.py` (conformance kits: cases, recorded count, idempotency, no skips, run in process and serialized by consumers at N-1 and N); `binding.py`, `codec.py` and `serializable.py` (what may cross a contract); `adapter.py` (public test adapters); `manifest.py` (`module.toml`); `registry.py` and `interface_errors.py`; - `module.toml`, this module's manifest, and `interface-record.json` (beside `pyproject.toml`), the record of every interface contract that `python -m tools.contracts.interface_record` writes and `python -m tools.contracts.interface_compat` compares with the last release. Module pack: [`docs/modules/contracts.md`](../../docs/modules/contracts.md). diff --git a/packages/contracts/src/orvex/contracts/adapter.py b/packages/contracts/src/orvex/contracts/adapter.py new file mode 100644 index 00000000..b42ee055 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/adapter.py @@ -0,0 +1,52 @@ +"""Public test adapters (#486 G4): implementations of an interface contract, shipped with it, +that a module's own tests use in place of the modules that provide the contracts it requires. + +An adapter lives in `..adapter` (`orvex.contracts.registry`) and must pass +its contract's whole kit (`packages/contracts/tests`), so a module runs its own operations +standalone, with only `orvex-contracts` installed and no module that only consumes it. +`required_adapters` gives a module the adapter of each contract its manifest requires, or +refuses with MODULE_ADAPTER_MISSING when one is missing or at an incompatible version. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass + +from .interface import InterfaceContract +from .interface_errors import InterfaceContractError +from .manifest import Manifest + + +@dataclass(frozen=True, slots=True) +class ContractAdapter: + """A public test adapter: `make` builds a fresh implementation of `contract` in full.""" + + contract: InterfaceContract + make: Callable[[], object] + + +def required_adapters( + manifest: Manifest, adapters: Mapping[str, ContractAdapter] +) -> dict[str, ContractAdapter]: + """The adapter of every contract `manifest` requires, by contract name.""" + found: dict[str, ContractAdapter] = {} + missing: list[str] = [] + for required in manifest.requires: + adapter = adapters.get(required.contract) + if adapter is None: + missing.append(f"{required.contract} {required.version}: no public test adapter") + elif not adapter.contract.version.provides(required.version): + missing.append( + f"{required.contract} {required.version}: the adapter is at " + f"{adapter.contract.version}" + ) + else: + found[required.contract] = adapter + if missing: + raise InterfaceContractError( + "MODULE_ADAPTER_MISSING", + f"{manifest.module} requires " + "; ".join(missing), + {"module": manifest.module}, + ) + return found diff --git a/packages/contracts/src/orvex/contracts/binding.py b/packages/contracts/src/orvex/contracts/binding.py index ffdb193a..d65697bf 100644 --- a/packages/contracts/src/orvex/contracts/binding.py +++ b/packages/contracts/src/orvex/contracts/binding.py @@ -1,11 +1,15 @@ """The two bindings an implementation is reached through (#486 G5): in process, and serialized as an independent process would reach it. -Through either binding only the contract's operations are callable, and an error crosses only -when it is the contract's error type with a code the operation declares -(INTERFACE_ERROR_UNDECLARED otherwise). Serialized, the arguments, the result and the error's -code, message and details each pass through the codec, so a live object is refused -(INTERFACE_VALUE_NOT_SERIALIZABLE) and nothing is shared with the caller. +Through either binding only the contract's operations are callable, each with its call +envelope first (`orvex.contracts.envelope`), validated before the implementation runs: a +refused call never reaches it (ContractCallRefused). The binding serves the implementation at +the version it provides (`provided`, the contract's version by default). An error crosses only +when it is the contract's error type with a code the operation declares, or the machinery's own +`InterfaceContractError`; anything else raised is INTERFACE_ERROR_UNDECLARED. Serialized, the +envelope, the arguments, the result and the error's code, message and details each pass +through the codec, so a live object is refused (INTERFACE_VALUE_NOT_SERIALIZABLE) and nothing +is shared with the caller. """ from __future__ import annotations @@ -15,22 +19,46 @@ from typing import Any from .codec import Codec +from .envelope import envelope_refusal from .interface import InterfaceContract -from .interface_errors import InterfaceContractError +from .interface_errors import ContractCallRefused, InterfaceContractError +from .version import Version class Bound: - """An implementation of `contract`, reached through one binding.""" + """An implementation of `contract` at version `provided`, reached through one binding.""" - __slots__ = ("_codec", "_contract", "_implementation") + __slots__ = ("_codec", "_contract", "_implementation", "_provided") def __init__( - self, contract: InterfaceContract, implementation: object, *, serialized: bool + self, + contract: InterfaceContract, + implementation: object, + *, + serialized: bool, + provided: Version | None = None, ) -> None: + version = contract.version if provided is None else provided + if not contract.version.provides(version): + raise ContractCallRefused( + "CONTRACT_VERSION_UNSUPPORTED", + f"contract {contract.name} {contract.version} cannot serve an implementation " + f"at {version}", + {"contract": contract.name, "provided": str(version)}, + ) self._contract = contract self._implementation = implementation + self._provided = version self._codec = Codec(contract.value_classes()) if serialized else None + @property + def contract(self) -> InterfaceContract: + return self._contract + + @property + def provided(self) -> Version: + return self._provided + @property def serialized(self) -> bool: return self._codec is not None @@ -40,8 +68,13 @@ def __getattr__(self, name: str) -> Callable[..., Any]: raise AttributeError(f"contract {self._contract.name} has no operation {name!r}") return functools.partial(self.call, name) - def call(self, operation: str, /, *arguments: object, **keywords: object) -> Any: # noqa: ANN401 + def call( + self, operation: str, envelope: object, /, *arguments: object, **keywords: object + ) -> Any: # noqa: ANN401 - the operation's result declared = self._contract.operation(operation) + refusal = envelope_refusal(envelope, self._contract, declared, self._provided) + if refusal is not None: + raise refusal method = getattr(self._implementation, operation, None) if not callable(method): raise InterfaceContractError( @@ -50,21 +83,31 @@ def call(self, operation: str, /, *arguments: object, **keywords: object) -> Any ) codec = self._codec if codec is not None: - arguments, keywords = _crossed(codec, (arguments, keywords)) + envelope, arguments, keywords = _crossed(codec, (envelope, arguments, keywords)) try: - result = method(*arguments, **keywords) + result = method(envelope, *arguments, **keywords) except self._contract.error_type as error: code = getattr(error, "code", None) if code not in declared.errors: - raise InterfaceContractError( - "INTERFACE_ERROR_UNDECLARED", - f"{self._contract.name}.{operation} raised {code!r}, which it does not declare", - ) from error + raise self._undeclared(operation, repr(code)) from (None if codec else error) if codec is None: raise raise self._rebuilt(codec, error) from None + except InterfaceContractError as error: + if codec is None: + raise + raise _copied(codec, error) from None + except Exception as error: # noqa: BLE001 - fail closed: nothing undeclared crosses + raise self._undeclared(operation, type(error).__name__) from (None if codec else error) return result if codec is None else _crossed(codec, result) + def _undeclared(self, operation: str, what: str) -> InterfaceContractError: + return InterfaceContractError( + "INTERFACE_ERROR_UNDECLARED", + f"{self._contract.name}.{operation} raised {what}, which it does not declare", + {"contract": self._contract.name, "operation": operation}, + ) + def _rebuilt(self, codec: Codec, error: Exception) -> Exception: parts = tuple(getattr(error, name, None) for name in ("code", "message", "details")) code, message, details = _crossed(codec, parts) @@ -72,9 +115,16 @@ def _rebuilt(self, codec: Codec, error: Exception) -> Exception: raise InterfaceContractError( "INTERFACE_VALUE_NOT_SERIALIZABLE", f"a {type(error).__name__} needs a message and details to cross the contract", - ) from error + ) from None return self._contract.error_type(code, message, details) +def _copied(codec: Codec, error: InterfaceContractError) -> InterfaceContractError: + """The machinery's own error, crossed as values: its class, code, message and details.""" + code, message, details = _crossed(codec, (error.code, error.message, error.details)) + kind = ContractCallRefused if type(error) is ContractCallRefused else InterfaceContractError + return kind(code, message, details) + + def _crossed(codec: Codec, value: object) -> Any: # noqa: ANN401 - the value decoded back return codec.decode(codec.encode(value)) diff --git a/packages/contracts/src/orvex/contracts/envelope.py b/packages/contracts/src/orvex/contracts/envelope.py new file mode 100644 index 00000000..6d297725 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/envelope.py @@ -0,0 +1,103 @@ +"""The call envelope (#486 §2 item 6): what every cross-module call carries, and why a binding +refuses one before the implementation runs. + +Every operation of an interface contract takes a `CallEnvelope` as its first argument: + +- `contract_name` and `contract_version`, the contract and the `MAJOR.MINOR` version the caller + speaks (never the module's version or a physical format's version); +- `correlation_id`, propagated end to end; +- `actor`, the effective actor or a verifiable reference to it; +- `project_id` and `revision`, the project and the revision the call acts on; +- `idempotency_key`, required on every operation that changes state. + +A field that is absent, not a string or blank, a key missing on an operation that changes +state, or an envelope for another contract is refused with CONTRACT_ENVELOPE_INVALID. A version +the provider cannot serve is refused with CONTRACT_VERSION_UNSUPPORTED: another major version, +or an operation the provider's version or the caller's version does not have. A caller at a +newer minor version is served for the operations the provider has (N against N-1). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from .interface_errors import ContractCallRefused +from .version import VERSION, Version + +if TYPE_CHECKING: + from .interface import InterfaceContract, Operation + +REQUIRED = ( + "contract_name", + "contract_version", + "correlation_id", + "actor", + "project_id", + "revision", +) +IDEMPOTENCY_KEY = "idempotency_key" + + +@dataclass(frozen=True, slots=True) +class CallEnvelope: + """The envelope of one call; see the module's docstring for each field.""" + + contract_name: str + contract_version: str + correlation_id: str + actor: str + project_id: str + revision: str + idempotency_key: str | None = None + + +def _present(value: object) -> bool: + return type(value) is str and bool(value.strip()) + + +def envelope_refusal( + envelope: object, contract: InterfaceContract, operation: Operation, provided: Version +) -> ContractCallRefused | None: + """Why a call of `operation`, served at version `provided`, is refused; None when it is not.""" + where = {"contract": contract.name, "operation": operation.name} + + def invalid(field: str, text: str) -> ContractCallRefused: + return ContractCallRefused( + "CONTRACT_ENVELOPE_INVALID", + f"{contract.name}.{operation.name}: {text}", + {**where, "field": field}, + ) + + if type(envelope) is not CallEnvelope: + return invalid("envelope", "a call carries a CallEnvelope as its first argument") + for field in REQUIRED: + if not _present(getattr(envelope, field, None)): + return invalid(field, f"the envelope has no {field}") + if operation.changes_state and not _present(envelope.idempotency_key): + return invalid(IDEMPOTENCY_KEY, "an operation that changes state needs an idempotency key") + if envelope.idempotency_key is not None and type(envelope.idempotency_key) is not str: + return invalid(IDEMPOTENCY_KEY, "an idempotency key is a string") + if envelope.contract_name != contract.name: + return invalid("contract_name", f"the envelope is for {envelope.contract_name!r}") + match = VERSION.fullmatch(envelope.contract_version) + if match is None: + return invalid("contract_version", f"{envelope.contract_version!r} is not MAJOR.MINOR") + spoken = Version(int(match[1]), int(match[2])) + since = operation.since or Version(contract.version.major, 0) + problem = ( + f"the caller speaks {spoken} and the provider serves {provided}" + if spoken.major != provided.major + else f"{operation.name} exists from {since}; the provider serves {provided}" + if not provided.provides(since) + else f"{operation.name} exists from {since}; the caller speaks {spoken}" + if not spoken.provides(since) + else None + ) + if problem is None: + return None + return ContractCallRefused( + "CONTRACT_VERSION_UNSUPPORTED", + f"{contract.name}.{operation.name}: {problem}", + {**where, "requested": str(spoken), "provided": str(provided)}, + ) diff --git a/packages/contracts/src/orvex/contracts/interface.py b/packages/contracts/src/orvex/contracts/interface.py index d5eaee49..ebd6f0e6 100644 --- a/packages/contracts/src/orvex/contracts/interface.py +++ b/packages/contracts/src/orvex/contracts/interface.py @@ -2,7 +2,8 @@ operations, inputs, outputs, errors, capabilities and version. An interface contract is a `typing.Protocol` whose public methods are the operations (their -annotations are the inputs and outputs), plus an `InterfaceContract` that gives the contract a +annotations are the inputs and outputs; the first parameter of each is the call envelope, +`envelope: CallEnvelope`), plus an `InterfaceContract` that gives the contract a name and a `major.minor` version, groups the operations into capabilities, and lists the error codes each operation may raise through the contract's error type (built as `error_type(code, message, details)`). The declaration is checked when it is built: a refused @@ -16,12 +17,13 @@ import typing from dataclasses import dataclass +from .envelope import CallEnvelope from .interface_errors import InterfaceContractError from .serializable import crossing_problem, type_name +from .version import Version NAME = re.compile(r"[a-z][a-z0-9]*(-[a-z0-9]+)*") ERROR_CODE = re.compile(r"[A-Z][A-Z0-9]*(_[A-Z0-9]+)*") -VERSION = re.compile(r"([1-9][0-9]*)\.(0|[1-9][0-9]*)") # A capability is either implemented in full or not provided: this code can never stand in for # an operation a contract declares (#486 G1). UNSUPPORTED_OPERATION = "UNSUPPORTED_OPERATION" @@ -32,37 +34,17 @@ } -@dataclass(frozen=True, slots=True, order=True) -class Version: - """A contract version: within a major version the contract only grows (append-only).""" - - major: int - minor: int - - @classmethod - def parse(cls, text: str) -> Version: - match = VERSION.fullmatch(text) - if match is None: - raise InterfaceContractError( - "INTERFACE_VERSION_INVALID", f"{text!r} is not a version: expected MAJOR.MINOR" - ) - return cls(int(match[1]), int(match[2])) - - def provides(self, required: Version) -> bool: - """An implementation at this version serves a consumer that requires `required`.""" - return self.major == required.major and self.minor >= required.minor - - def __str__(self) -> str: - return f"{self.major}.{self.minor}" - - @dataclass(frozen=True, slots=True) class Operation: - """One operation: its method name, the capability it belongs to and the codes it raises.""" + """One operation: its method name, the capability it belongs to, the codes it raises, + whether it changes state (then every call carries an idempotency key) and the version it + exists from (`None`: the first version of the contract's major).""" name: str capability: str errors: tuple[str, ...] = () + changes_state: bool = False + since: Version | None = None @dataclass(frozen=True, slots=True) @@ -88,6 +70,10 @@ def operation(self, name: str) -> Operation: "INTERFACE_OPERATION_UNKNOWN", f"contract {self.name} has no operation {name!r}" ) + def since(self, operation: Operation) -> Version: + """The version `operation` exists from.""" + return operation.since or Version(self.version.major, 0) + def operations_of(self, capability: str) -> tuple[Operation, ...]: return tuple(o for o in self.operations if o.capability == capability) @@ -110,6 +96,8 @@ def describe(self) -> dict[str, object]: signature = self.signature(operation.name) operations[operation.name] = { "capability": operation.capability, + "changes_state": operation.changes_state, + "since": str(self.since(operation)), "errors": sorted(operation.errors), "parameters": [ { @@ -174,6 +162,13 @@ def _check_names(contract: InterfaceContract) -> None: raise _invalid( contract, f"{operation.name}: capability {operation.capability!r} is undeclared" ) + if operation.since is not None and not ( + operation.since.major == contract.version.major and operation.since <= contract.version + ): + raise _invalid( + contract, + f"{operation.name}: since {operation.since} is not a version of {contract.version}", + ) unknown = sorted(set(operation.errors) - set(contract.errors)) if unknown: raise _invalid( @@ -200,6 +195,16 @@ def _check_operations(contract: InterfaceContract) -> None: if not inspect.isfunction(member): raise _invalid(contract, f"{operation.name}: an operation is a plain method") signature = _signature(contract.protocol, operation.name) + first = next(iter(signature.parameters.values()), None) + if ( + first is None + or first.name != "envelope" + or first.annotation is not CallEnvelope + or first.kind is inspect.Parameter.KEYWORD_ONLY + ): + raise _invalid( + contract, f"{operation.name}: the first parameter is `envelope: CallEnvelope`" + ) for parameter in signature.parameters.values(): if parameter.kind not in KINDS: raise _invalid(contract, f"{operation.name}: *{parameter.name} cannot be described") diff --git a/packages/contracts/src/orvex/contracts/interface_errors.py b/packages/contracts/src/orvex/contracts/interface_errors.py index 6ef17be8..2a8257b8 100644 --- a/packages/contracts/src/orvex/contracts/interface_errors.py +++ b/packages/contracts/src/orvex/contracts/interface_errors.py @@ -1,10 +1,13 @@ -"""The error of the interface-contract machinery: a stable code, a message and details. +"""The errors of the interface-contract machinery: a stable code, a message and details. -Codes: INTERFACE_CONTRACT_INVALID, INTERFACE_VERSION_INVALID, INTERFACE_TYPE_NOT_SERIALIZABLE, +`ContractCallRefused` is the typed refusal of one call before the implementation runs: +CONTRACT_ENVELOPE_INVALID and CONTRACT_VERSION_UNSUPPORTED (`orvex.contracts.envelope`). +Every other code is an `InterfaceContractError`: INTERFACE_CONTRACT_INVALID, +INTERFACE_VERSION_INVALID, INTERFACE_TYPE_NOT_SERIALIZABLE, INTERFACE_UNSUPPORTED_OPERATION_DECLARED, INTERFACE_OPERATION_UNKNOWN, INTERFACE_OPERATION_MISSING, INTERFACE_VALUE_NOT_SERIALIZABLE, INTERFACE_ERROR_UNDECLARED, -KIT_INVALID, KIT_CASE_COUNT_CHANGED, KIT_OPERATION_UNCOVERED, KIT_FAILED and -MODULE_MANIFEST_INVALID. +KIT_INVALID, KIT_CASE_COUNT_CHANGED, KIT_OPERATION_UNCOVERED, KIT_IDEMPOTENCY_UNCOVERED, +KIT_FAILED, MODULE_MANIFEST_INVALID and MODULE_ADAPTER_MISSING. """ from __future__ import annotations @@ -20,3 +23,7 @@ def __init__(self, code: str, message: str, details: Mapping[str, str] | None = self.code = code self.message = message self.details = dict(details or {}) + + +class ContractCallRefused(InterfaceContractError): + """A call refused by its binding before the implementation runs (#486 §2 item 6, G3).""" diff --git a/packages/contracts/src/orvex/contracts/kit.py b/packages/contracts/src/orvex/contracts/kit.py index ab46c16c..0f40f190 100644 --- a/packages/contracts/src/orvex/contracts/kit.py +++ b/packages/contracts/src/orvex/contracts/kit.py @@ -2,10 +2,12 @@ with their recorded count and no skip list. A kit belongs to one interface contract. Each case has a stable id, the operation it exercises, -its expectation in words (the guarantee the compatibility check holds to, #486 G3) and a check -that receives the implementation through a binding and raises when the guarantee fails. The -kit records how many cases it has; a kit whose cases differ from that count, or that leaves an -operation without a case, is refused when it is built. +its expectation in words (the guarantee the compatibility check holds to, #486 G3), the version +it exists from, and either a check that receives a `KitClient` and raises when the guarantee +fails, or an `Idempotent` pair for an operation that changes state: the call is sent twice with +one idempotency key, and the effect must happen exactly once (#486 G3). The kit records how +many cases it has; a kit whose cases differ from that count, that leaves an operation without a +case, or an operation that changes state without an idempotency case, is refused when built. """ from __future__ import annotations @@ -14,19 +16,31 @@ from collections.abc import Callable from dataclasses import dataclass -from .binding import Bound +from .envelope import CallEnvelope from .interface import InterfaceContract from .interface_errors import InterfaceContractError +from .kit_client import KitClient +from .version import Version CASE_ID = re.compile(r"[a-z][a-z0-9]*([.-][a-z0-9]+)*") +@dataclass(frozen=True, slots=True) +class Idempotent: + """An idempotency check: `call` sends the operation with the envelope it is given, and + `observe` reads the effect through the contract's other operations.""" + + call: Callable[[KitClient, CallEnvelope], object] + observe: Callable[[KitClient], object] + + @dataclass(frozen=True, slots=True) class KitCase: id: str operation: str expectation: str - check: Callable[[Bound], None] + check: Callable[[KitClient], None] | Idempotent + since: Version | None = None @dataclass(frozen=True, slots=True) @@ -40,7 +54,7 @@ def __post_init__(self) -> None: operations = {operation.name for operation in self.contract.operations} seen: set[str] = set() for case in self.cases: - problem = ( + problem = (_case_problem(self, case) if case.operation in operations else None) or ( "its id is not lowercase, dotted or kebab-case" if not CASE_ID.fullmatch(case.id) else "its id is used twice" @@ -61,13 +75,46 @@ def __post_init__(self) -> None: raise InterfaceContractError( "KIT_OPERATION_UNCOVERED", f"kit {name}: operations {uncovered} have no case" ) + idempotent = {c.operation for c in self.cases if isinstance(c.check, Idempotent)} + unchecked = sorted( + o.name for o in self.contract.operations if o.changes_state and o.name not in idempotent + ) + if unchecked: + raise InterfaceContractError( + "KIT_IDEMPOTENCY_UNCOVERED", + f"kit {name}: operations {unchecked} change state and have no idempotency case", + ) if len(self.cases) != self.recorded: raise InterfaceContractError( "KIT_CASE_COUNT_CHANGED", f"kit {name} records {self.recorded} cases and has {len(self.cases)}", ) + def since(self, case: KitCase) -> Version: + """The version `case` exists from.""" + return case.since or Version(self.contract.version.major, 0) + def describe(self) -> dict[str, object]: """The kit as plain data: what the compatibility check compares (#486 G3).""" - cases = {c.id: {"operation": c.operation, "expectation": c.expectation} for c in self.cases} + cases = { + case.id: { + "operation": case.operation, + "expectation": case.expectation, + "idempotency": isinstance(case.check, Idempotent), + "since": str(self.since(case)), + } + for case in self.cases + } return {"recorded": self.recorded, "cases": cases} + + +def _case_problem(kit: Kit, case: KitCase) -> str | None: + """What is wrong with a case of a declared operation: its version or its idempotency.""" + contract = kit.contract + operation = contract.operation(case.operation) + since = kit.since(case) + if not (contract.version.provides(since) and since >= contract.since(operation)): + return f"since {since} is not between {contract.since(operation)} and {contract.version}" + if isinstance(case.check, Idempotent) and not operation.changes_state: + return "an idempotency case needs an operation that changes state" + return None diff --git a/packages/contracts/src/orvex/contracts/kit_client.py b/packages/contracts/src/orvex/contracts/kit_client.py new file mode 100644 index 00000000..1c32932f --- /dev/null +++ b/packages/contracts/src/orvex/contracts/kit_client.py @@ -0,0 +1,65 @@ +"""What a kit case calls an implementation through (#486 G1, G3): its binding, with a complete +envelope made for each call at the run's consumer version. + +`client.(...)` sends a new envelope, with a fresh idempotency key when the operation +changes state; `client.envelope(operation)` and `client.send(operation, envelope, ...)` let a +case send one envelope twice (the idempotency check) or a changed one. +""" + +from __future__ import annotations + +import dataclasses +import functools +from collections.abc import Callable +from typing import Any + +from .binding import Bound +from .envelope import CallEnvelope +from .version import Version + + +class KitClient: + """One case's caller: a consumer at `consumer` reaching the implementation through `bound`.""" + + __slots__ = ("_bound", "_case", "_consumer", "_keys") + + def __init__(self, bound: Bound, consumer: Version, case: str) -> None: + self._bound = bound + self._consumer = consumer + self._case = case + self._keys = 0 + + @property + def consumer(self) -> Version: + return self._consumer + + def envelope(self, operation: str, **changes: Any) -> CallEnvelope: # noqa: ANN401 + """A complete envelope for one call of `operation`, with `changes` applied (a case may + plant any value, a missing field included).""" + key = None + if self._bound.contract.operation(operation).changes_state: + self._keys += 1 + key = f"kit-{self._case}-{self._keys}" + envelope = CallEnvelope( + contract_name=self._bound.contract.name, + contract_version=str(self._consumer), + correlation_id=f"kit-{self._case}", + actor="kit-actor", + project_id="kit-project", + revision="kit-revision", + idempotency_key=key, + ) + return dataclasses.replace(envelope, **changes) + + def send( + self, operation: str, envelope: object, /, *arguments: object, **keywords: object + ) -> Any: # noqa: ANN401 - the operation's result + return self._bound.call(operation, envelope, *arguments, **keywords) + + def __getattr__(self, name: str) -> Callable[..., Any]: + if name.startswith("_") or name not in {o.name for o in self._bound.contract.operations}: + raise AttributeError(f"contract {self._bound.contract.name} has no operation {name!r}") + return functools.partial(self._new, name) + + def _new(self, operation: str, /, *arguments: object, **keywords: object) -> Any: # noqa: ANN401 + return self.send(operation, self.envelope(operation), *arguments, **keywords) diff --git a/packages/contracts/src/orvex/contracts/kit_run.py b/packages/contracts/src/orvex/contracts/kit_run.py index 4a043517..3695ea1b 100644 --- a/packages/contracts/src/orvex/contracts/kit_run.py +++ b/packages/contracts/src/orvex/contracts/kit_run.py @@ -1,83 +1,264 @@ -"""Running a conformance kit (#486 G1, G5): every case, in process and serialized, each on a fresh -implementation, with nothing skipped and the same results through both bindings.""" +"""Running a conformance kit (#486 G1, G3, G5): every case, each on a fresh implementation, in +process and serialized, by consumers at N-1 and N, with nothing skipped. + +A run serves the implementation at the version it provides (`provided`) to a consumer at +version `consumer`, which runs the cases that exist at its version. A case that exists at the +provided version must pass. A newer case (a consumer at N against a provider at N-1) must pass +or be refused with CONTRACT_VERSION_UNSUPPORTED: never a wrong answer. Each run also sends the +envelope cases the contract implies, which no implementation can pass by itself: a call missing +each envelope field, or without its idempotency key where it changes state, is refused with +CONTRACT_ENVELOPE_INVALID, and a call at another major version with +CONTRACT_VERSION_UNSUPPORTED, both before the implementation is reached. +""" from __future__ import annotations -from collections.abc import Callable +import functools +from collections.abc import Callable, Iterator from dataclasses import dataclass +from typing import Literal from .binding import Bound +from .envelope import IDEMPOTENCY_KEY, REQUIRED +from .interface import InterfaceContract, Operation from .interface_errors import InterfaceContractError -from .kit import Kit +from .kit import Idempotent, Kit, KitCase +from .kit_client import KitClient +from .version import Version + +Outcome = Literal["passed", "refused", "failed"] +UNSUPPORTED = "CONTRACT_VERSION_UNSUPPORTED" @dataclass(frozen=True, slots=True) class CaseResult: id: str - passed: bool + outcome: Outcome detail: str + @property + def passed(self) -> bool: + return self.outcome == "passed" + @dataclass(frozen=True, slots=True) class KitReport: contract: str serialized: bool + consumer: Version + provided: Version results: tuple[CaseResult, ...] + envelope_results: tuple[CaseResult, ...] -def binding_name(serialized: bool) -> str: - return "serialized" if serialized else "in process" +class _Reached: + """Wraps an implementation and records whether any operation of it was reached.""" + def __init__(self, implementation: object) -> None: + self.implementation = implementation + self.reached: list[str] = [] -def run_kit(kit: Kit, make: Callable[[], object], *, serialized: bool) -> KitReport: - results: list[CaseResult] = [] - for case in kit.cases: - bound = Bound(kit.contract, make(), serialized=serialized) + def __getattr__(self, name: str) -> object: + self.reached.append(name) + return getattr(self.implementation, name) + + +def binding_name(report: KitReport) -> str: + binding = "serialized" if report.serialized else "in process" + return f"{binding}, consumer {report.consumer}, provider {report.provided}" + + +def _code(error: BaseException) -> str | None: + found: BaseException | None = error + while found is not None: + if isinstance(found, InterfaceContractError): + return found.code + found = found.__cause__ + return None + + +def _idempotent(client: KitClient, case: KitCase, check: Idempotent) -> None: + before = check.observe(client) + envelope = client.envelope(case.operation) + first = check.call(client, envelope) + once = check.observe(client) + second = check.call(client, envelope) + twice = check.observe(client) + if once == before: + raise AssertionError("KIT_IDEMPOTENCY_NO_EFFECT: the first call changed nothing") + if twice != once: + raise AssertionError( + "KIT_IDEMPOTENCY_BROKEN: one idempotency key had its effect twice: " + f"{once!r} became {twice!r}" + ) + if second != first: + raise AssertionError( + f"KIT_IDEMPOTENCY_BROKEN: the repeated call answered {second!r}, first {first!r}" + ) + + +def _outcome(run: Callable[[], None]) -> tuple[Outcome, str]: + try: + run() + except KeyboardInterrupt, SystemExit: + raise + except BaseException as error: # noqa: BLE001 - a failure or a skip: the kit has no skips + outcome: Outcome = "refused" if _code(error) == UNSUPPORTED else "failed" + return outcome, f"{type(error).__name__}: {error}" + return "passed", "" + + +def _envelope_changes( + contract: InterfaceContract, operation: Operation, consumer: Version +) -> Iterator[tuple[str, str, dict[str, str | None]]]: + """Each envelope case of one operation: its name, the code it expects, the change made.""" + for field in REQUIRED: + yield f"{field}-missing", "CONTRACT_ENVELOPE_INVALID", {field: None} + yield f"{field}-blank", "CONTRACT_ENVELOPE_INVALID", {field: " "} + if operation.changes_state: + yield f"{IDEMPOTENCY_KEY}-missing", "CONTRACT_ENVELOPE_INVALID", {IDEMPOTENCY_KEY: None} + yield "other-contract", "CONTRACT_ENVELOPE_INVALID", {"contract_name": f"{contract.name}-x"} + for major in (consumer.major + 1, consumer.major - 1): + if major >= 1: + yield f"major-{major}", UNSUPPORTED, {"contract_version": f"{major}.0"} + + +def _envelope_cases( + kit: Kit, consumer: Version +) -> Iterator[tuple[Operation, str, str, dict[str, str | None]]]: + """Each envelope case a consumer at `consumer` sends: its operation, id, code and change.""" + for operation in kit.contract.operations: + if consumer.provides(kit.contract.since(operation)): + for name, expected, change in _envelope_changes(kit.contract, operation, consumer): + yield operation, f"envelope.{operation.name}.{name}", expected, change + + +def _envelope_results( + kit: Kit, make: Callable[[], object], serialized: bool, consumer: Version, provided: Version +) -> list[CaseResult]: + results = [] + for operation, case_id, expected, change in _envelope_cases(kit, consumer): + spy = _Reached(make()) + bound = Bound(kit.contract, spy, serialized=serialized, provided=provided) + client = KitClient(bound, consumer, case_id) + envelope = client.envelope(operation.name, **change) try: - case.check(bound) - except KeyboardInterrupt, SystemExit: - raise - except BaseException as error: # noqa: BLE001 - a failure or a skip: the kit has no skips - results.append(CaseResult(case.id, False, f"{type(error).__name__}: {error}")) + client.send(operation.name, envelope) + except InterfaceContractError as error: + if error.code == expected and not spy.reached: + results.append(CaseResult(case_id, "passed", "")) + continue + detail = f"{error.code}, reached {spy.reached}" + except Exception as error: # noqa: BLE001 - anything but the refusal fails the case + detail = f"{type(error).__name__}: {error}, reached {spy.reached}" else: - results.append(CaseResult(case.id, True, "")) - return KitReport(kit.contract.name, serialized, tuple(results)) + detail = f"accepted, reached {spy.reached}" + results.append(CaseResult(case_id, "failed", f"expected {expected}; {detail}")) + return results + + +def run_kit( + kit: Kit, + make: Callable[[], object], + *, + serialized: bool, + consumer: Version | None = None, + provided: Version | None = None, +) -> KitReport: + """One run of the kit: one binding, one consumer version, one provided version.""" + served = kit.contract.version if provided is None else provided + speaking = kit.contract.version if consumer is None else consumer + if not kit.contract.version.provides(speaking): + raise InterfaceContractError( + "KIT_INVALID", f"kit {kit.contract.name} has no consumer at {speaking}" + ) + results = [] + for case in kit.cases: + if not speaking.provides(kit.since(case)): + continue + bound = Bound(kit.contract, make(), serialized=serialized, provided=served) + client = KitClient(bound, speaking, case.id) + check = case.check + if isinstance(check, Idempotent): + outcome, detail = _outcome(functools.partial(_idempotent, client, case, check)) + else: + outcome, detail = _outcome(functools.partial(check, client)) + results.append(CaseResult(case.id, outcome, detail)) + envelope = _envelope_results(kit, make, serialized, speaking, served) + return KitReport( + kit.contract.name, serialized, speaking, served, tuple(results), tuple(envelope) + ) def kit_problems(kit: Kit, report: KitReport) -> list[str]: - """Every way `report` falls short of the whole kit passing.""" - binding = binding_name(report.serialized) + """Every way `report` falls short of the whole kit passing for its consumer and provider.""" + binding = binding_name(report) + expected = [case for case in kit.cases if report.consumer.provides(kit.since(case))] ran = [result.id for result in report.results] problems = [ f"KIT_RUN_INCOMPLETE: {case.id} did not run ({binding})" - for case in kit.cases + for case in expected if case.id not in ran ] - if len(ran) != kit.recorded and not problems: + if len(ran) != len(expected) and not problems: problems.append( - f"KIT_RUN_INCOMPLETE: {len(ran)} results for {kit.recorded} cases ({binding})" + f"KIT_RUN_INCOMPLETE: {len(ran)} results for {len(expected)} cases ({binding})" ) + since = {case.id: kit.since(case) for case in kit.cases} + for result in report.results: + newer = result.id in since and not report.provided.provides(since[result.id]) + if result.outcome == "failed" or (result.outcome == "refused" and not newer): + problems.append(f"KIT_CASE_FAILED: {result.id} ({binding}): {result.detail}") + sent = [result.id for result in report.envelope_results] + problems += [ + f"KIT_RUN_INCOMPLETE: {case_id} did not run ({binding})" + for _, case_id, _, _ in _envelope_cases(kit, report.consumer) + if case_id not in sent + ] problems += [ - f"KIT_CASE_FAILED: {result.id} ({binding}): {result.detail}" - for result in report.results + f"KIT_ENVELOPE_CASE_FAILED: {result.id} ({binding}): {result.detail}" + for result in report.envelope_results if not result.passed ] return problems -def verify_kit(kit: Kit, make: Callable[[], object]) -> tuple[KitReport, KitReport]: - """Run the whole kit in process and serialized; raise KIT_FAILED unless every case passes - through both bindings with the same results.""" - in_process = run_kit(kit, make, serialized=False) - serialized = run_kit(kit, make, serialized=True) - problems = kit_problems(kit, in_process) + kit_problems(kit, serialized) - outcomes = [[(r.id, r.passed) for r in report.results] for report in (in_process, serialized)] - if outcomes[0] != outcomes[1]: - problems.insert( - 0, "KIT_BINDINGS_DIFFER: the serialized run differs from the run in process" - ) +def consumers(contract: InterfaceContract, provided: Version) -> tuple[Version, ...]: + """The consumer versions a provider at `provided` is run against: N-1, N and the newest.""" + versions = {provided, contract.version} + previous = provided.previous() + if previous is not None: + versions.add(previous) + return tuple(sorted(versions)) + + +def verify_kit( + kit: Kit, make: Callable[[], object], provided: Version | None = None +) -> tuple[KitReport, ...]: + """Run the whole kit in process and serialized, by every consumer of `consumers`; raise + KIT_FAILED unless every run passes and both bindings give the same results.""" + served = kit.contract.version if provided is None else provided + reports: list[KitReport] = [] + problems: list[str] = [] + for consumer in consumers(kit.contract, served): + pair = [ + run_kit(kit, make, serialized=serialized, consumer=consumer, provided=served) + for serialized in (False, True) + ] + outcomes = [ + [(r.id, r.outcome) for r in (*report.results, *report.envelope_results)] + for report in pair + ] + if outcomes[0] != outcomes[1]: + problems.append( + f"KIT_BINDINGS_DIFFER: the serialized run differs from the run in process " + f"(consumer {consumer}, provider {served})" + ) + for report in pair: + problems += kit_problems(kit, report) + reports += pair if problems: raise InterfaceContractError( "KIT_FAILED", "; ".join(problems), {"contract": kit.contract.name} ) - return in_process, serialized + return tuple(reports) diff --git a/packages/contracts/src/orvex/contracts/manifest.py b/packages/contracts/src/orvex/contracts/manifest.py index 1d328483..66b1048d 100644 --- a/packages/contracts/src/orvex/contracts/manifest.py +++ b/packages/contracts/src/orvex/contracts/manifest.py @@ -28,8 +28,9 @@ from dataclasses import dataclass from typing import Any -from .interface import NAME, Version +from .interface import NAME from .interface_errors import InterfaceContractError +from .version import Version MODULE = re.compile(r"[a-z]+") DISTRIBUTION = re.compile(r"[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?") diff --git a/packages/contracts/src/orvex/contracts/registry.py b/packages/contracts/src/orvex/contracts/registry.py index b0fdb362..56d25f10 100644 --- a/packages/contracts/src/orvex/contracts/registry.py +++ b/packages/contracts/src/orvex/contracts/registry.py @@ -1,6 +1,7 @@ -"""The interface contracts and kits a contracts package ships, found by their place: -`..interface` holds `InterfaceContract` values and `..kit` holds -`Kit` values (ADR-0033).""" +"""The interface contracts, kits and public test adapters a contracts package ships, found by +their place: `..interface` holds `InterfaceContract` values, +`..kit` holds `Kit` values and `..adapter` holds +`ContractAdapter` values (ADR-0033).""" from __future__ import annotations @@ -9,6 +10,7 @@ import pkgutil from types import ModuleType +from .adapter import ContractAdapter from .interface import InterfaceContract from .interface_errors import InterfaceContractError from .kit import Kit @@ -22,11 +24,15 @@ def kits(package: ModuleType) -> dict[str, Kit]: return _declared(package, "kit", Kit) -def _name(value: InterfaceContract | Kit) -> str: +def adapters(package: ModuleType) -> dict[str, ContractAdapter]: + return _declared(package, "adapter", ContractAdapter) + + +def _name(value: InterfaceContract | Kit | ContractAdapter) -> str: return value.name if isinstance(value, InterfaceContract) else value.contract.name -def _declared[Declared: (InterfaceContract, Kit)]( +def _declared[Declared: (InterfaceContract, Kit, ContractAdapter)]( package: ModuleType, place: str, kind: type[Declared] ) -> dict[str, Declared]: found: dict[str, Declared] = {} diff --git a/packages/contracts/src/orvex/contracts/version.py b/packages/contracts/src/orvex/contracts/version.py new file mode 100644 index 00000000..97d6baea --- /dev/null +++ b/packages/contracts/src/orvex/contracts/version.py @@ -0,0 +1,38 @@ +"""A contract version, `MAJOR.MINOR` (#486 G3): within a major version a contract only grows.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from .interface_errors import InterfaceContractError + +VERSION = re.compile(r"([1-9][0-9]*)\.(0|[1-9][0-9]*)") + + +@dataclass(frozen=True, slots=True, order=True) +class Version: + """A contract version: within a major version the contract only grows (append-only).""" + + major: int + minor: int + + @classmethod + def parse(cls, text: str) -> Version: + match = VERSION.fullmatch(text) + if match is None: + raise InterfaceContractError( + "INTERFACE_VERSION_INVALID", f"{text!r} is not a version: expected MAJOR.MINOR" + ) + return cls(int(match[1]), int(match[2])) + + def provides(self, required: Version) -> bool: + """An implementation at this version serves a consumer that requires `required`.""" + return self.major == required.major and self.minor >= required.minor + + def previous(self) -> Version | None: + """The previous minor version of the same major (N-1), if there is one.""" + return Version(self.major, self.minor - 1) if self.minor else None + + def __str__(self) -> str: + return f"{self.major}.{self.minor}" diff --git a/tools/checks/module_dependencies.py b/tools/checks/module_dependencies.py index 3bf2beae..396670a2 100644 --- a/tools/checks/module_dependencies.py +++ b/tools/checks/module_dependencies.py @@ -10,7 +10,8 @@ `module.toml` and the distribution disagree; MODULE_MANIFEST_MISSING: no `module.toml`; MODULE_TEST_IMPORTS_REPOSITORY: a test imports `tests`, `conformance` or `tools`; MODULE_TEST_DEPENDENCY_UNDECLARED / _UNUSED: the tests' own dependencies against -`dependencies.tests`. +`dependencies.tests`; MODULE_ADAPTER_MISSING: a contract the module requires has no public test +adapter in the installed contracts package, so the module could not run without its providers. """ from __future__ import annotations @@ -24,14 +25,15 @@ from collections.abc import Callable, Iterable, Mapping from pathlib import Path +from orvex.contracts.adapter import required_adapters +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.manifest import Manifest, canonical, parse_manifest +from orvex.contracts.registry import adapters + REPOSITORY_HELPERS = frozenset({"tests", "conformance", "tools"}) Owner = Callable[[str], "str | None"] -def canonical(distribution: str) -> str: - return re.sub(r"[-_.]+", "-", distribution).lower() - - def imported_modules(source: str) -> set[str]: """The absolute imports of one file, by their full dotted names.""" found: set[str] = set() @@ -124,6 +126,15 @@ def suite_problems( return problems +def adapter_problems(manifest: Manifest, namespace: str) -> list[str]: + """MODULE_ADAPTER_MISSING when a required contract has no public test adapter installed.""" + try: + required_adapters(manifest, adapters(importlib.import_module(f"{namespace}.contracts"))) + except InterfaceContractError as error: + return [str(error)] + return [] + + def file_owners() -> dict[Path, str]: """Every installed file, with the distribution that installed it.""" owners: dict[Path, str] = {} @@ -160,16 +171,13 @@ def owner_of(name: str) -> str | None: def main(argv: list[str]) -> int: namespace, name, tests = argv - manifest_module = importlib.import_module(f"{namespace}.contracts.manifest") distribution = importlib.metadata.distribution(name) paths = [Path(str(distribution.locate_file(f))) for f in distribution.files or ()] manifests = [p for p in paths if p.name == "module.toml" and p.parent.parent.name == namespace] if len(manifests) != 1: print(f"MODULE_MANIFEST_MISSING: {name} ships {len(manifests)} module.toml, expected one") return 1 - manifest = manifest_module.parse_manifest( - manifests[0].read_text("utf-8"), manifests[0].parent.name - ) + manifest = parse_manifest(manifests[0].read_text("utf-8"), manifests[0].parent.name) declared = [ re.match(r"[A-Za-z0-9._-]+", requirement)[0] # type: ignore[index] for requirement in distribution.requires or () @@ -186,6 +194,7 @@ def main(argv: list[str]) -> int: for path in sorted(Path(tests).rglob("*.py")) } problems += suite_problems(name, by_file, owner_of, manifest.runtime, manifest.tests) + problems += adapter_problems(manifest, namespace) for problem in problems: print(problem) print(f"module dependencies of {name}: {len(problems)} problem(s)") diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index 6743329d..1f2aca77 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -726,7 +726,7 @@ reason = "needs a ClickHouse 26.8 LTS server (CORPUS_CLICKHOUSE), started by the workflow = "interfaces.yml" job = "interface-contracts" kind = "check" -required = false +required = true contexts = ["interface-contracts"] digest = "db80658d940be1da" setup = ["python -m pip install --quiet pip==26.3"] @@ -745,7 +745,7 @@ local = true workflow = "interfaces.yml" job = "standalone" kind = "check" -required = false +required = true contexts = ["standalone"] digest = "bcb329418caea5d2" setup = ["python -m pip install --quiet pip==26.3"] diff --git a/tools/contracts/interface_compat.py b/tools/contracts/interface_compat.py index ccf22111..472239c8 100644 --- a/tools/contracts/interface_compat.py +++ b/tools/contracts/interface_compat.py @@ -6,12 +6,15 @@ (the newest `v` tag HEAD contains): within a major version a contract only grows. These fail, unless the major version went up: a removed contract (INTERFACE_CONTRACT_REMOVED), a lowered version (INTERFACE_VERSION_LOWERED), a removed operation (INTERFACE_OPERATION_REMOVED), -a changed signature or capability of an operation (INTERFACE_SIGNATURE_CHANGED), a removed -capability (INTERFACE_CAPABILITY_REMOVED), a removed or renamed error code -(INTERFACE_ERROR_CODE_REMOVED), a removed kit case (INTERFACE_KIT_CASE_REMOVED) or a changed -expectation (INTERFACE_KIT_EXPECTATION_CHANGED). Allowed: new contracts, operations, -capabilities, error codes, kit cases and new parameters with a default. No release tag fails -closed (INTERFACE_RELEASE_UNKNOWN); a release without a record released no interface contract. +a changed signature, capability, state change or version of an operation +(INTERFACE_SIGNATURE_CHANGED), a removed capability (INTERFACE_CAPABILITY_REMOVED), a removed +or renamed error code (INTERFACE_ERROR_CODE_REMOVED), a removed kit case +(INTERFACE_KIT_CASE_REMOVED), a changed expectation (INTERFACE_KIT_EXPECTATION_CHANGED), and an +operation or kit case added since the release that claims to exist at the released version +(INTERFACE_ADDITION_UNVERSIONED). Allowed: new contracts, and new operations, capabilities, +error codes, kit cases and parameters with a default at a new minor version. No release tag +fails closed (INTERFACE_RELEASE_UNKNOWN); a release without a record released no interface +contract. Running N against N-1 in both directions is the kit's (`orvex.contracts.kit_run`). """ from __future__ import annotations @@ -23,7 +26,7 @@ from pathlib import Path from typing import Any -from orvex.contracts.interface import Version +from orvex.contracts.version import Version from tools.contracts.interface_record import RECORD, current_record from tools.repository import REPOSITORY_ROOT @@ -42,6 +45,11 @@ def _operation_problems( f"INTERFACE_SIGNATURE_CHANGED: {where} moved from capability " f"{old['capability']} to {new['capability']}" ) + for part in ("changes_state", "since"): + if new[part] != old[part]: + problems.append( + f"INTERFACE_SIGNATURE_CHANGED: {where} {part} is {new[part]}, was {old[part]}" + ) if new["returns"] != old["returns"]: problems.append( f"INTERFACE_SIGNATURE_CHANGED: {where} returns {new['returns']}, was {old['returns']}" @@ -62,8 +70,28 @@ def _shape(parameter: dict[str, Any]) -> tuple[str, str, str]: return parameter["name"], parameter["kind"], parameter["type"] +def _unversioned(name: str, old: dict[str, Any], new: dict[str, Any]) -> list[str]: + """Additions that claim to exist at the released version: a consumer at N-1 would expect + them from a provider that does not have them.""" + released = Version.parse(old["version"]) + added = [ + (f"{name}.{operation}", entry["since"]) + for operation, entry in sorted(new["operations"].items()) + if operation not in old["operations"] + ] + [ + (f"{name}: kit case {case}", entry["since"]) + for case, entry in sorted(new["kit"]["cases"].items()) + if case not in old["kit"]["cases"] + ] + return [ + f"INTERFACE_ADDITION_UNVERSIONED: {where} is new since {released} and claims {since}" + for where, since in added + if Version.parse(since) <= released + ] + + def _contract_problems(name: str, old: dict[str, Any], new: dict[str, Any]) -> list[str]: - problems = [] + problems = _unversioned(name, old, new) for operation, before in sorted(old["operations"].items()): after = new["operations"].get(operation) if after is None: diff --git a/tools/repo/main-ruleset.json b/tools/repo/main-ruleset.json index a8a62eb8..ee19bd65 100644 --- a/tools/repo/main-ruleset.json +++ b/tools/repo/main-ruleset.json @@ -106,6 +106,14 @@ { "context": "security-scenarios", "integration_id": 15368 + }, + { + "context": "interface-contracts", + "integration_id": 15368 + }, + { + "context": "standalone", + "integration_id": 15368 } ] } From 0a9183b5c6f66827ac8651ac685f2936bf172d93 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 02:10:07 -0400 Subject: [PATCH 05/21] refactor(contracts): one place decides what an operation is and when it exists Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- packages/contracts/src/orvex/contracts/binding.py | 2 +- packages/contracts/src/orvex/contracts/envelope.py | 2 +- packages/contracts/src/orvex/contracts/interface.py | 4 ++++ packages/contracts/src/orvex/contracts/kit_client.py | 2 +- packages/contracts/tests/test_envelope.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/contracts/src/orvex/contracts/binding.py b/packages/contracts/src/orvex/contracts/binding.py index d65697bf..e0fb2145 100644 --- a/packages/contracts/src/orvex/contracts/binding.py +++ b/packages/contracts/src/orvex/contracts/binding.py @@ -64,7 +64,7 @@ def serialized(self) -> bool: return self._codec is not None def __getattr__(self, name: str) -> Callable[..., Any]: - if name.startswith("_") or name not in {o.name for o in self._contract.operations}: + if not self._contract.offers(name): raise AttributeError(f"contract {self._contract.name} has no operation {name!r}") return functools.partial(self.call, name) diff --git a/packages/contracts/src/orvex/contracts/envelope.py b/packages/contracts/src/orvex/contracts/envelope.py index 6d297725..2a56d256 100644 --- a/packages/contracts/src/orvex/contracts/envelope.py +++ b/packages/contracts/src/orvex/contracts/envelope.py @@ -84,7 +84,7 @@ def invalid(field: str, text: str) -> ContractCallRefused: if match is None: return invalid("contract_version", f"{envelope.contract_version!r} is not MAJOR.MINOR") spoken = Version(int(match[1]), int(match[2])) - since = operation.since or Version(contract.version.major, 0) + since = contract.since(operation) problem = ( f"the caller speaks {spoken} and the provider serves {provided}" if spoken.major != provided.major diff --git a/packages/contracts/src/orvex/contracts/interface.py b/packages/contracts/src/orvex/contracts/interface.py index ebd6f0e6..afb0202d 100644 --- a/packages/contracts/src/orvex/contracts/interface.py +++ b/packages/contracts/src/orvex/contracts/interface.py @@ -70,6 +70,10 @@ def operation(self, name: str) -> Operation: "INTERFACE_OPERATION_UNKNOWN", f"contract {self.name} has no operation {name!r}" ) + def offers(self, name: str) -> bool: + """Whether `name` is one of the contract's operations (a public attribute of a binding).""" + return not name.startswith("_") and any(o.name == name for o in self.operations) + def since(self, operation: Operation) -> Version: """The version `operation` exists from.""" return operation.since or Version(self.version.major, 0) diff --git a/packages/contracts/src/orvex/contracts/kit_client.py b/packages/contracts/src/orvex/contracts/kit_client.py index 1c32932f..e2cf1456 100644 --- a/packages/contracts/src/orvex/contracts/kit_client.py +++ b/packages/contracts/src/orvex/contracts/kit_client.py @@ -57,7 +57,7 @@ def send( return self._bound.call(operation, envelope, *arguments, **keywords) def __getattr__(self, name: str) -> Callable[..., Any]: - if name.startswith("_") or name not in {o.name for o in self._bound.contract.operations}: + if not self._bound.contract.offers(name): raise AttributeError(f"contract {self._bound.contract.name} has no operation {name!r}") return functools.partial(self._new, name) diff --git a/packages/contracts/tests/test_envelope.py b/packages/contracts/tests/test_envelope.py index 644de3cd..945400d4 100644 --- a/packages/contracts/tests/test_envelope.py +++ b/packages/contracts/tests/test_envelope.py @@ -141,6 +141,6 @@ def test_a_binding_never_serves_a_version_the_contract_does_not_describe( assert raised.value.code == "CONTRACT_VERSION_UNSUPPORTED" -def test_the_envelope_holds_the_required_fields_and_the_key() -> None: +def test_every_field_of_the_envelope_is_validated() -> None: assert dataclasses.fields(CallEnvelope)[-1].name == "idempotency_key" assert [f.name for f in dataclasses.fields(CallEnvelope)][:-1] == list(REQUIRED) From 219c8e877873fa313d2170734451743a23a2dd64 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 03:36:02 -0400 Subject: [PATCH 06/21] ci(repo): the interface checks stay optional until main's protection requires them tools/ci/protection.py compares the required checks with main's live rulesets, so a check becomes required only after the Owner adds it there. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- tools/ci/checks.toml | 4 ++-- tools/repo/main-ruleset.json | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index 1f2aca77..6743329d 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -726,7 +726,7 @@ reason = "needs a ClickHouse 26.8 LTS server (CORPUS_CLICKHOUSE), started by the workflow = "interfaces.yml" job = "interface-contracts" kind = "check" -required = true +required = false contexts = ["interface-contracts"] digest = "db80658d940be1da" setup = ["python -m pip install --quiet pip==26.3"] @@ -745,7 +745,7 @@ local = true workflow = "interfaces.yml" job = "standalone" kind = "check" -required = true +required = false contexts = ["standalone"] digest = "bcb329418caea5d2" setup = ["python -m pip install --quiet pip==26.3"] diff --git a/tools/repo/main-ruleset.json b/tools/repo/main-ruleset.json index ee19bd65..a8a62eb8 100644 --- a/tools/repo/main-ruleset.json +++ b/tools/repo/main-ruleset.json @@ -106,14 +106,6 @@ { "context": "security-scenarios", "integration_id": 15368 - }, - { - "context": "interface-contracts", - "integration_id": 15368 - }, - { - "context": "standalone", - "integration_id": 15368 } ] } From 7b46943b8f897333dbab57655f88ec1ec51389e8 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 03:55:33 -0400 Subject: [PATCH 07/21] fix(contracts): close the review findings on refusals, compatibility, codec and assembly - only a binding creates a refusal; anything else an implementation raises is undeclared, and the kit counts no refusal found down a cause chain; - compatibility compares each operation's error codes and default values; - one import scanner serves the module and consumer rules and the dynamic-import note; - models cross field by field, Arrow tables as IPC (pyarrow declared); - assembly chooses among several implementations by configuration and checks construction, signatures (envelope included) and the operations of the provided version. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- ...-interface-contracts-kits-and-manifests.md | 6 +- packages/contracts/pyproject.toml | 2 +- .../contracts/src/orvex/contracts/adapter.py | 5 + .../contracts/src/orvex/contracts/binding.py | 21 +--- .../contracts/src/orvex/contracts/codec.py | 25 +++- .../contracts/src/orvex/contracts/envelope.py | 10 +- .../src/orvex/contracts/interface.py | 51 +++++++- .../contracts/src/orvex/contracts/kit_run.py | 15 +-- .../contracts/src/orvex/contracts/manifest.py | 5 +- .../contracts/src/orvex/contracts/module.toml | 4 +- .../src/orvex/contracts/serializable.py | 18 ++- packages/contracts/tests/test_adapter.py | 10 ++ packages/contracts/tests/test_binding.py | 91 +++++++++++++- packages/contracts/tests/test_interface.py | 49 +++++++- packages/contracts/tests/test_kit.py | 40 +++++- packages/contracts/tests/test_manifest.py | 2 +- packages/server/src/orvex/server/assembly.py | 113 +++++++++++++---- packages/server/tests/test_assembly.py | 116 +++++++++++++++--- tests/tools/test_checks.py | 12 ++ tests/tools/test_consumer_boundaries.py | 21 +++- tests/tools/test_interface_compat.py | 42 ++++++- tools/checks/boundaries.py | 18 +-- tools/checks/consumers.py | 29 +---- tools/checks/imports.py | 42 +++++++ tools/checks/module_dependencies.py | 14 ++- tools/checks/structure.py | 9 +- tools/contracts/interface_compat.py | 30 +++-- 27 files changed, 632 insertions(+), 168 deletions(-) create mode 100644 tools/checks/imports.py diff --git a/docs/adr/0033-interface-contracts-kits-and-manifests.md b/docs/adr/0033-interface-contracts-kits-and-manifests.md index a98db4a1..c4c2cdbc 100644 --- a/docs/adr/0033-interface-contracts-kits-and-manifests.md +++ b/docs/adr/0033-interface-contracts-kits-and-manifests.md @@ -6,12 +6,12 @@ ADR-0004 makes modules talk only through contracts, but its contracts are data contracts: they say what crosses a boundary, never which operations a module offers. On 2026-11-14 no module imported another module's internals, yet `server`, the conformance runners, `tools/` and the repository tests used 65 internal paths of the modules (#486 §1), the boundaries check exempted `server` as the composition root, and no module had ever been installed or tested outside the repository. Nothing proved that another implementation of a module could replace it. ## Decision -1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version (`orvex.contracts.version`), groups the operations into capabilities, lists the error codes each may raise through the contract's error type, marks the operations that change state and the version each exists from. Only values cross: the annotated inputs and outputs are plain values, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`, its kit in `orvex.contracts..kit` and its public test adapter, when it has one, in `orvex.contracts..adapter` (`orvex.contracts.registry`). +1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version (`orvex.contracts.version`), groups the operations into capabilities, lists the error codes each may raise through the contract's error type, marks the operations that change state and the version each exists from. Only values cross: the annotated inputs and outputs are plain values, Arrow tables, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. The serializing codec keeps each exactly: a model field by field (its own JSON would turn an IANA zone into an offset), an Arrow table as Arrow IPC. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`, its kit in `orvex.contracts..kit` and its public test adapter, when it has one, in `orvex.contracts..adapter` (`orvex.contracts.registry`). 2. **The call envelope.** Every operation takes a `CallEnvelope` (`orvex.contracts.envelope`) as its first argument: `contract_name` and `contract_version` (the contract's version, never the module's or a physical format's), `correlation_id`, the effective `actor` or a verifiable reference to it, `project_id` and `revision`, and an `idempotency_key`, required on every operation that changes state. Every binding (`orvex.contracts.binding`) validates it before the implementation runs and fails closed with a typed `ContractCallRefused`: CONTRACT_ENVELOPE_INVALID for a missing, blank or mistyped field, a missing key or another contract's envelope; CONTRACT_VERSION_UNSUPPORTED for another major version or an operation that the provider's or the caller's version does not have. A caller at a newer minor version is served for the operations the provider has. 3. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise, their expectation in words and the version they exist from, a recorded case count and no skip list. A kit leaves no operation without a case, and each operation that changes state has an idempotency case: the call is sent twice with one key and its effect must happen exactly once. The runner (`orvex.contracts.kit_run`, `kit_client`, `codec`) runs every case on a fresh implementation, once in process and once through the serializing binding, by a consumer at N-1 and at N, and adds the envelope cases the contract implies (each missing field and key, another contract, another major version), which must be refused before the implementation is reached. A provider at N-1 runs the kit of N: the cases of its version pass, the newer ones pass or are refused with CONTRACT_VERSION_UNSUPPORTED, never answered wrongly. An implementation that passes its contract's whole kit may replace the module. 4. **Consumers know contracts, not modules** (amends rule 3). `server` stays the only composition root, but only `server/assembly.py` names implementations, and only through each module's public `api` package. Every other consumer (`server`, the conformance runners, `tools/`, the repository tests) imports only `orvex.contracts`. `tools/checks/consumers.py` enforces it, in report mode until #486 step 7 makes it blocking and removes the `server` exemption. -5. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` attribute that implements each) and requires (version, capabilities), and the distributions its code and its tests import. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation or more than one, a version that does not provide the required one, a missing capability, or an implementation lacking an operation of what it claims. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. -6. **Compatibility of behavior.** The record of every interface contract and its kit (`packages/contracts/interface-record.json`, `tools/contracts/interface_record.py`) is compared with the record at the last product release (`tools/contracts/interface_compat.py`): within a major version operations, signatures, capabilities, error codes and kit cases with their expectations may only be added, each addition existing only from a version after the release. A break needs a new major version (rule 6 of ADR-0004 applies to it). Rule 3 runs N against N-1 in both directions: each module's own tests run its kit by consumers at N-1 and N, and an implementation at N-1 runs the current kit with its own version as `provided`. The job that builds a module's last released implementation for that run arrives with the first released interface contract (#486 step 2); until then no released implementation exists. +5. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` class that implements each) and requires (version, capabilities), and the distributions its code and its tests import. A module is named by its package path under the namespace, so an alternative implementation (`alternatives.result_cache_ipc`) has a manifest too. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation, or several and the configuration chose none (or chose one that does not provide it); a version that does not provide the required one; a missing capability; an implementation that is not a class built from configuration by keyword arguments, lacks an operation of what it claims at its version, or whose method differs from the contract's signature, envelope included. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. `module.toml` does not replace `contracts.toml`: `contracts.toml` declares the language-neutral data contracts a module produces and consumes, by their `vN` versions (ADR-0004 rule 6, `versioning.md` §3), and the server checks them as before; `module.toml` declares the interface contracts, by `MAJOR.MINOR`, and the module's dependencies. Merging them would change the checked data-contract declarations of every module at once, which is outside #486. +6. **Compatibility of behavior.** The record of every interface contract and its kit (`packages/contracts/interface-record.json`, `tools/contracts/interface_record.py`) is compared with the record at the last product release (`tools/contracts/interface_compat.py`): within a major version operations, capabilities, error codes and kit cases with their expectations may only be added, each addition existing only from a version after the release; the signature of an existing operation (default values included) and the error codes it raises never change. A break needs a new major version (rule 6 of ADR-0004 applies to it). Rule 3 runs N against N-1 in both directions: each module's own tests run its kit by consumers at N-1 and N, and an implementation at N-1 runs the current kit with its own version as `provided`. The job that builds a module's last released implementation for that run arrives with the first released interface contract (#486 step 2); until then no released implementation exists. 7. **Standalone modules.** `tools/ci/standalone.py` builds a module's sdist and, from it, its wheel, installs it in a clean environment outside the repository with only `orvex-contracts` beside it, checks that its imports equal its declared dependencies, that its tests import no repository helper and that every contract it requires has a public test adapter (`orvex.contracts.adapter`, itself passing the contract's whole kit), and runs its own tests there, where a skipped test fails. A module thus starts and runs its own operations without the modules that only consume it. ## Alternatives considered diff --git a/packages/contracts/pyproject.toml b/packages/contracts/pyproject.toml index 67c55300..67a51820 100644 --- a/packages/contracts/pyproject.toml +++ b/packages/contracts/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.14" -dependencies = ["pydantic==2.14.3"] +dependencies = ["pyarrow==26.0.1", "pydantic==2.14.3"] [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/contracts/src/orvex/contracts/adapter.py b/packages/contracts/src/orvex/contracts/adapter.py index b42ee055..a095a920 100644 --- a/packages/contracts/src/orvex/contracts/adapter.py +++ b/packages/contracts/src/orvex/contracts/adapter.py @@ -36,6 +36,11 @@ def required_adapters( adapter = adapters.get(required.contract) if adapter is None: missing.append(f"{required.contract} {required.version}: no public test adapter") + elif adapter.contract.name != required.contract: + missing.append( + f"{required.contract} {required.version}: the adapter found implements " + f"{adapter.contract.name}" + ) elif not adapter.contract.version.provides(required.version): missing.append( f"{required.contract} {required.version}: the adapter is at " diff --git a/packages/contracts/src/orvex/contracts/binding.py b/packages/contracts/src/orvex/contracts/binding.py index e0fb2145..c9a70aee 100644 --- a/packages/contracts/src/orvex/contracts/binding.py +++ b/packages/contracts/src/orvex/contracts/binding.py @@ -3,10 +3,12 @@ Through either binding only the contract's operations are callable, each with its call envelope first (`orvex.contracts.envelope`), validated before the implementation runs: a -refused call never reaches it (ContractCallRefused). The binding serves the implementation at -the version it provides (`provided`, the contract's version by default). An error crosses only -when it is the contract's error type with a code the operation declares, or the machinery's own -`InterfaceContractError`; anything else raised is INTERFACE_ERROR_UNDECLARED. Serialized, the +refused call never reaches it (ContractCallRefused, which only a binding creates). The binding +serves the implementation at the version it provides (`provided`, the contract's version by +default). An error crosses only when it is the contract's error type with a code the operation +declares; anything else the implementation raises, a refusal or another machinery error +included, is INTERFACE_ERROR_UNDECLARED: a nested call's refusal crosses only as a code the +operation declares. Serialized, the envelope, the arguments, the result and the error's code, message and details each pass through the codec, so a live object is refused (INTERFACE_VALUE_NOT_SERIALIZABLE) and nothing is shared with the caller. @@ -93,10 +95,6 @@ def call( if codec is None: raise raise self._rebuilt(codec, error) from None - except InterfaceContractError as error: - if codec is None: - raise - raise _copied(codec, error) from None except Exception as error: # noqa: BLE001 - fail closed: nothing undeclared crosses raise self._undeclared(operation, type(error).__name__) from (None if codec else error) return result if codec is None else _crossed(codec, result) @@ -119,12 +117,5 @@ def _rebuilt(self, codec: Codec, error: Exception) -> Exception: return self._contract.error_type(code, message, details) -def _copied(codec: Codec, error: InterfaceContractError) -> InterfaceContractError: - """The machinery's own error, crossed as values: its class, code, message and details.""" - code, message, details = _crossed(codec, (error.code, error.message, error.details)) - kind = ContractCallRefused if type(error) is ContractCallRefused else InterfaceContractError - return kind(code, message, details) - - def _crossed(codec: Codec, value: object) -> Any: # noqa: ANN401 - the value decoded back return codec.decode(codec.encode(value)) diff --git a/packages/contracts/src/orvex/contracts/codec.py b/packages/contracts/src/orvex/contracts/codec.py index dd4b18d1..49d87189 100644 --- a/packages/contracts/src/orvex/contracts/codec.py +++ b/packages/contracts/src/orvex/contracts/codec.py @@ -2,7 +2,10 @@ Every value becomes tagged JSON: exact types survive (a tuple stays a tuple, `bool` is never an `int`), a `Decimal` keeps its scale, an integer of any width stays exact, a `datetime` keeps its -zone (an IANA zone by its key) and its `fold`. Only the classes the contract declares +zone (an IANA zone by its key) and its `fold`. A pydantic model crosses field by field through +the same tree and is validated again when rebuilt, so nothing inside it is weakened (its own +JSON would turn an IANA zone into a fixed offset). An Arrow table crosses as Arrow IPC, which +keeps every type, NULL and time zone of its schema. Only the classes the contract declares (`InterfaceContract.value_classes`) are built back, and only of exactly that class. Anything else is refused with INTERFACE_VALUE_NOT_SERIALIZABLE: a live object never crosses. """ @@ -18,6 +21,7 @@ from typing import Any from zoneinfo import ZoneInfo +import pyarrow as pa from pydantic import BaseModel from .interface_errors import InterfaceContractError @@ -69,6 +73,8 @@ def _tree(self, value: object) -> Any: # noqa: ANN401 - a JSON tree return {"$time": value.isoformat()} if kind is timedelta and isinstance(value, timedelta): return {"$timedelta": [value.days, value.seconds, value.microseconds]} + if kind is pa.Table and isinstance(value, pa.Table): + return {"$arrow": base64.b64encode(_ipc(value)).decode()} if kind in (tuple, list) and isinstance(value, (tuple, list)): return {f"${kind.__name__}": [self._tree(item) for item in value]} if kind is dict and isinstance(value, dict): @@ -85,7 +91,8 @@ def _object(self, value: object) -> Any: # noqa: ANN401 - a JSON tree if isinstance(value, Enum): return {"$enum": name, "member": value.name} if isinstance(value, BaseModel): - return {"$model": name, "json": value.model_dump_json()} + fields = {f: self._tree(getattr(value, f)) for f in type(value).model_fields} + return {"$model": name, "fields": fields} if not dataclasses.is_dataclass(value): raise _refused(f"a {name} is not a value") fields = {f.name: self._tree(getattr(value, f.name)) for f in dataclasses.fields(value)} @@ -100,7 +107,7 @@ def _value(self, tree: Any) -> object: # noqa: ANN401 - a JSON tree payload = tree[tag] try: return self._tagged(tag, payload, tree) - except (ValueError, TypeError, KeyError) as error: + except (ValueError, TypeError, KeyError, ArithmeticError) as error: raise _refused(f"not an encoded {tag}: {error}") from None def _tagged(self, tag: str, payload: Any, tree: dict[str, Any]) -> object: # noqa: ANN401 @@ -120,6 +127,8 @@ def _tagged(self, tag: str, payload: Any, tree: dict[str, Any]) -> object: # no if tag == "$timedelta": days, seconds, microseconds = payload return timedelta(days=days, seconds=seconds, microseconds=microseconds) + if tag == "$arrow": + return pa.ipc.open_stream(base64.b64decode(payload, validate=True)).read_all() if tag == "$tuple": return tuple(self._value(item) for item in payload) if tag == "$list": @@ -132,12 +141,20 @@ def _tagged(self, tag: str, payload: Any, tree: dict[str, Any]) -> object: # no if tag == "$enum" and issubclass(kind, Enum): return kind[tree["member"]] if tag == "$model" and issubclass(kind, BaseModel): - return kind.model_validate_json(tree["json"]) + fields = {name: self._value(item) for name, item in tree["fields"].items()} + return kind.model_validate(fields, by_alias=False, by_name=True) if tag == "$dataclass" and dataclasses.is_dataclass(kind): return kind(**{name: self._value(item) for name, item in tree["fields"].items()}) raise _refused(f"{tag} does not match {payload}") +def _ipc(table: pa.Table) -> bytes: + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + return bytes(sink.getvalue()) + + def _zone(value: datetime) -> dict[str, object]: zone = value.tzinfo found: dict[str, object] = {"fold": value.fold} if value.fold else {} diff --git a/packages/contracts/src/orvex/contracts/envelope.py b/packages/contracts/src/orvex/contracts/envelope.py index 2a56d256..3a0b8420 100644 --- a/packages/contracts/src/orvex/contracts/envelope.py +++ b/packages/contracts/src/orvex/contracts/envelope.py @@ -22,8 +22,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from .interface_errors import ContractCallRefused -from .version import VERSION, Version +from .interface_errors import ContractCallRefused, InterfaceContractError +from .version import Version if TYPE_CHECKING: from .interface import InterfaceContract, Operation @@ -80,10 +80,10 @@ def invalid(field: str, text: str) -> ContractCallRefused: return invalid(IDEMPOTENCY_KEY, "an idempotency key is a string") if envelope.contract_name != contract.name: return invalid("contract_name", f"the envelope is for {envelope.contract_name!r}") - match = VERSION.fullmatch(envelope.contract_version) - if match is None: + try: + spoken = Version.parse(envelope.contract_version) + except InterfaceContractError: return invalid("contract_version", f"{envelope.contract_version!r} is not MAJOR.MINOR") - spoken = Version(int(match[1]), int(match[2])) since = contract.since(operation) problem = ( f"the caller speaks {spoken} and the provider serves {provided}" diff --git a/packages/contracts/src/orvex/contracts/interface.py b/packages/contracts/src/orvex/contracts/interface.py index afb0202d..cc738b06 100644 --- a/packages/contracts/src/orvex/contracts/interface.py +++ b/packages/contracts/src/orvex/contracts/interface.py @@ -85,6 +85,21 @@ def signature(self, name: str) -> inspect.Signature: """The operation's signature without `self`, its annotations resolved.""" return _signature(self.protocol, self.operation(name).name) + def mismatch(self, name: str, implementation: type) -> str | None: + """Why the method `name` of `implementation` does not have the operation's signature + (parameters with their kinds, types and defaults, and the result), or None.""" + expected = self.signature(name) + method = inspect.getattr_static(implementation, name, None) + if not inspect.isfunction(method): + return f"{name} is not a plain method" + try: + actual = _function_signature(method) + except (NameError, TypeError) as error: + return f"{name}: its annotations cannot be resolved: {error}" + if _shape(actual) != _shape(expected): + return f"{name}{_text(actual)} is not {name}{_text(expected)}" + return None + def value_classes(self) -> frozenset[type]: """The enums, models and dataclasses that may cross this contract.""" classes: set[type] = set() @@ -108,7 +123,7 @@ def describe(self) -> dict[str, object]: "name": parameter.name, "kind": KINDS[parameter.kind], "type": type_name(parameter.annotation), - "default": parameter.default is not inspect.Parameter.empty, + "default": _default(parameter), } for parameter in signature.parameters.values() ], @@ -227,7 +242,11 @@ def _check_operations(contract: InterfaceContract) -> None: def _signature(protocol: type, name: str) -> inspect.Signature: - method = vars(protocol)[name] + return _function_signature(vars(protocol)[name]) + + +def _function_signature(method: typing.Callable[..., object]) -> inspect.Signature: + """A method's signature without `self`, its annotations resolved.""" hints = typing.get_type_hints(method) signature = inspect.signature(method) parameters = [ @@ -241,3 +260,31 @@ def _signature(protocol: type, name: str) -> inspect.Signature: def _annotations(signature: inspect.Signature) -> list[object]: found: list[object] = [parameter.annotation for parameter in signature.parameters.values()] return [*found, signature.return_annotation] + + +def _default(parameter: inspect.Parameter) -> str | None: + """A parameter's default value as recorded text, or None when it has none.""" + return None if parameter.default is inspect.Parameter.empty else repr(parameter.default) + + +def _shape(signature: inspect.Signature) -> tuple[object, ...]: + """What a signature promises: each parameter's name, kind, type and default, and the result.""" + return ( + tuple( + ( + p.name, + p.kind, + type_name(p.annotation), + _default(p), + ) + for p in signature.parameters.values() + ), + type_name(signature.return_annotation), + ) + + +def _text(signature: inspect.Signature) -> str: + parameters = ", ".join( + f"{p.name}: {type_name(p.annotation)}" for p in signature.parameters.values() + ) + return f"({parameters}) -> {type_name(signature.return_annotation)}" diff --git a/packages/contracts/src/orvex/contracts/kit_run.py b/packages/contracts/src/orvex/contracts/kit_run.py index 3695ea1b..a7cac003 100644 --- a/packages/contracts/src/orvex/contracts/kit_run.py +++ b/packages/contracts/src/orvex/contracts/kit_run.py @@ -21,7 +21,7 @@ from .binding import Bound from .envelope import IDEMPOTENCY_KEY, REQUIRED from .interface import InterfaceContract, Operation -from .interface_errors import InterfaceContractError +from .interface_errors import ContractCallRefused, InterfaceContractError from .kit import Idempotent, Kit, KitCase from .kit_client import KitClient from .version import Version @@ -68,13 +68,10 @@ def binding_name(report: KitReport) -> str: return f"{binding}, consumer {report.consumer}, provider {report.provided}" -def _code(error: BaseException) -> str | None: - found: BaseException | None = error - while found is not None: - if isinstance(found, InterfaceContractError): - return found.code - found = found.__cause__ - return None +def _refused(error: BaseException) -> bool: + """A refusal by the binding itself: never one found down a `__cause__` chain, and never one + an implementation raised, which the binding turns into INTERFACE_ERROR_UNDECLARED.""" + return type(error) is ContractCallRefused and error.code == UNSUPPORTED def _idempotent(client: KitClient, case: KitCase, check: Idempotent) -> None: @@ -103,7 +100,7 @@ def _outcome(run: Callable[[], None]) -> tuple[Outcome, str]: except KeyboardInterrupt, SystemExit: raise except BaseException as error: # noqa: BLE001 - a failure or a skip: the kit has no skips - outcome: Outcome = "refused" if _code(error) == UNSUPPORTED else "failed" + outcome: Outcome = "refused" if _refused(error) else "failed" return outcome, f"{type(error).__name__}: {error}" return "passed", "" diff --git a/packages/contracts/src/orvex/contracts/manifest.py b/packages/contracts/src/orvex/contracts/manifest.py index 66b1048d..e9ed2200 100644 --- a/packages/contracts/src/orvex/contracts/manifest.py +++ b/packages/contracts/src/orvex/contracts/manifest.py @@ -1,7 +1,7 @@ """The module manifest `module.toml` (#486 §2.5): the interface contracts a module provides and requires, each at a version with its capabilities, and the distributions it depends on. - module = "store" + module = "store" # its package path under the namespace [dependencies] runtime = ["duckdb", "pyarrow"] # what its code imports, equal to its distribution's @@ -32,7 +32,8 @@ from .interface_errors import InterfaceContractError from .version import Version -MODULE = re.compile(r"[a-z]+") +# A module, or an alternative implementation, by its package path under the namespace. +MODULE = re.compile(r"[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*") DISTRIBUTION = re.compile(r"[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?") IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") diff --git a/packages/contracts/src/orvex/contracts/module.toml b/packages/contracts/src/orvex/contracts/module.toml index f648688c..473e97ab 100644 --- a/packages/contracts/src/orvex/contracts/module.toml +++ b/packages/contracts/src/orvex/contracts/module.toml @@ -1,7 +1,7 @@ # The manifest of this module (#486 §2.5, orvex.contracts.manifest): it implements no interface -# contract and requires none; the contract types depend on pydantic. +# contract and requires none; the contract types depend on pydantic, Arrow tables on pyarrow. module = "contracts" [dependencies] -runtime = ["pydantic"] +runtime = ["pyarrow", "pydantic"] tests = ["pytest"] diff --git a/packages/contracts/src/orvex/contracts/serializable.py b/packages/contracts/src/orvex/contracts/serializable.py index 33d10c15..e7e0ef3f 100644 --- a/packages/contracts/src/orvex/contracts/serializable.py +++ b/packages/contracts/src/orvex/contracts/serializable.py @@ -1,11 +1,11 @@ """Which annotated types may cross an interface contract (#486 G5): plain values, never live objects, so every operation can later run in another process (#156). -A value type is one of `SCALARS`, a `Literal` of booleans, integers, strings or `None`, a -union of value types, a parameterized `tuple`, `list` or `dict` with `str` keys, an `Enum`, a -pydantic model (the generated contract types) or a frozen dataclass whose fields are all value -types. Anything else (`Any`, `object`, callables, iterators, locks, connections, a mutable -dataclass, a bare `tuple`) is refused. +A value type is one of `SCALARS` (an Arrow table among them), a `Literal` of booleans, +integers, strings or `None`, a union of value types, a parameterized `tuple`, `list` or `dict` +with `str` keys, an `Enum`, or a pydantic model (the generated contract types) or frozen +dataclass whose fields are all value types. Anything else (`Any`, `object`, callables, +iterators, locks, connections, a mutable dataclass, a bare `tuple`) is refused. """ from __future__ import annotations @@ -17,6 +17,7 @@ from decimal import Decimal from enum import Enum +import pyarrow as pa from pydantic import BaseModel SCALARS: tuple[type, ...] = ( @@ -31,6 +32,7 @@ date, time, timedelta, + pa.Table, ) LITERAL_TYPES = (bool, int, str, type(None)) SEQUENCES = (tuple, list) @@ -88,9 +90,13 @@ def crossing_problem(annotation: object, classes: set[type]) -> str | None: def _class_problem(annotation: type, classes: set[type]) -> str | None: if annotation in classes: return None - if issubclass(annotation, (Enum, BaseModel)): + if issubclass(annotation, Enum): classes.add(annotation) return None + if issubclass(annotation, BaseModel): + classes.add(annotation) + fields = (field.annotation for field in annotation.model_fields.values()) + return _first(crossing_problem(field, classes) for field in fields) if dataclasses.is_dataclass(annotation): if not annotation.__dataclass_params__.frozen: # type: ignore[attr-defined] return f"{type_name(annotation)} is a mutable dataclass" diff --git a/packages/contracts/tests/test_adapter.py b/packages/contracts/tests/test_adapter.py index 1009c438..182a75ba 100644 --- a/packages/contracts/tests/test_adapter.py +++ b/packages/contracts/tests/test_adapter.py @@ -3,6 +3,7 @@ from __future__ import annotations +import dataclasses from decimal import Decimal import pytest @@ -62,6 +63,15 @@ def test_a_required_contract_without_an_adapter_is_refused( assert "billing requires ledger 1.0: no public test adapter" in raised.value.message +def test_an_adapter_of_another_contract_is_refused() -> None: + other = ContractAdapter(dataclasses.replace(LEDGER, name="journal"), MemoryLedger) + manifest = needing(Required("ledger", Version(1, 0), ("read",))) + with pytest.raises(InterfaceContractError) as raised: + required_adapters(manifest, {"ledger": other}) + assert raised.value.code == "MODULE_ADAPTER_MISSING" + assert "the adapter found implements journal" in raised.value.message + + @pytest.mark.parametrize("version", [Version(1, 2), Version(2, 0)]) def test_an_adapter_at_an_incompatible_version_is_refused(version: Version) -> None: with pytest.raises(InterfaceContractError) as raised: diff --git a/packages/contracts/tests/test_binding.py b/packages/contracts/tests/test_binding.py index 02a71d6f..c93a70cd 100644 --- a/packages/contracts/tests/test_binding.py +++ b/packages/contracts/tests/test_binding.py @@ -9,13 +9,14 @@ from enum import Enum from zoneinfo import ZoneInfo +import pyarrow as pa import pytest from orvex.contracts.binding import Bound from orvex.contracts.codec import Codec from orvex.contracts.envelope import CallEnvelope -from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.interface_errors import ContractCallRefused, InterfaceContractError from orvex.contracts.kit_run import run_kit, verify_kit -from pydantic import BaseModel +from pydantic import BaseModel, Field from .sample_contract import ( ENVELOPE, @@ -33,7 +34,29 @@ class Color(Enum): class Shape(BaseModel): - side: Decimal + side: Decimal = Field(gt=0) + + +class Moment(BaseModel): + """A model holding an IANA zone: its own JSON would turn the zone into a fixed offset.""" + + at: datetime + color: Color + readings: tuple[Reading, ...] + + +TABLE = pa.table( + { + "amount": pa.array([Decimal("12.50"), None], pa.decimal128(38, 2)), + "huge": pa.array([Decimal(2**126), Decimal(-(2**126))], pa.decimal128(38, 0)), + "at": pa.array( + [datetime(2026, 3, 8, 5, 30, tzinfo=UTC), None], pa.timestamp("us", "America/La_Paz") + ), + "count": pa.array([None, -(2**63)], pa.int64()), + "text": pa.array(["ñandú", None], pa.large_string()), + }, + metadata={"source": "test"}, +) @dataclass(frozen=True, slots=True) @@ -46,7 +69,7 @@ class ReadingChild(Reading): pass -CODEC = Codec(frozenset({Reading, Color, Shape, CallEnvelope})) +CODEC = Codec(frozenset({Reading, Color, Shape, Moment, CallEnvelope})) VALUES: list[object] = [ None, @@ -78,6 +101,13 @@ class ReadingChild(Reading): {"a": {"b": [Decimal("0.10")]}, "": None}, Color.RED, Shape(side=Decimal("2.50")), + Moment( + at=datetime(2026, 11, 1, 1, 30, fold=1, tzinfo=ZoneInfo("America/New_York")), + color=Color.RED, + readings=(READING,), + ), + TABLE, + TABLE.slice(0, 0), READING, ENVELOPE, (), @@ -96,6 +126,11 @@ def same(left: object, right: object) -> bool: if isinstance(left, dict): assert isinstance(right, dict) return left.keys() == right.keys() and all(same(left[k], right[k]) for k in left) + if isinstance(left, pa.Table): + assert isinstance(right, pa.Table) + return bool(left.equals(right, check_metadata=True)) + if isinstance(left, BaseModel): + return all(same(getattr(left, f), getattr(right, f)) for f in type(left).model_fields) if isinstance(left, Decimal): return str(left) == str(right) if isinstance(left, (datetime, time)): @@ -115,6 +150,26 @@ def test_every_value_comes_back_exactly(self, value: object) -> None: back = CODEC.decode(CODEC.encode(value)) assert same(value, back), (value, back) + def test_a_zone_inside_a_model_stays_an_iana_zone(self) -> None: + moment = Moment(at=READING.at, color=Color.RED, readings=()) + back = CODEC.decode(CODEC.encode(moment)) + assert isinstance(back, Moment) + assert back.at.tzinfo == ZoneInfo("America/La_Paz") + + def test_a_model_is_validated_again_when_rebuilt(self) -> None: + data = CODEC.encode(Shape(side=Decimal("2.50"))).replace(b"2.50", b"-2.50") + with pytest.raises(InterfaceContractError) as raised: + CODEC.decode(data) + assert raised.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" + + def test_an_arrow_table_keeps_its_types_nulls_and_zones(self) -> None: + back = CODEC.decode(CODEC.encode(TABLE)) + assert isinstance(back, pa.Table) + assert back.schema == TABLE.schema + assert back.column("amount").to_pylist() == [Decimal("12.50"), None] + assert back.column("huge").to_pylist()[0] == Decimal(2**126) + assert back.schema.field("at").type == pa.timestamp("us", "America/La_Paz") + def test_a_zone_comes_back_as_the_same_zone(self) -> None: back = CODEC.decode(CODEC.encode(READING.at)) assert isinstance(back, datetime) @@ -144,7 +199,14 @@ def test_a_live_or_undeclared_value_is_refused(self, value: object) -> None: @pytest.mark.parametrize( "data", - [b"not json", b'{"$unknown": 1}', b'{"$dataclass": "nowhere.Thing", "fields": {}}', b"{}"], + [ + b"not json", + b'{"$decimal": "two"}', + b'{"$arrow": "AAAA"}', + b'{"$unknown": 1}', + b'{"$dataclass": "nowhere.Thing", "fields": {}}', + b"{}", + ], ) def test_bytes_that_are_not_an_encoded_value_are_refused(self, data: bytes) -> None: with pytest.raises(InterfaceContractError) as raised: @@ -175,6 +237,20 @@ def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: raise KeyError(name) +class FakeRefusal(MemoryLedger): + """A planted violation: the implementation fakes the binding's version refusal.""" + + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: + raise ContractCallRefused("CONTRACT_VERSION_UNSUPPORTED", "faked", {}) + + +class LeakedMachineryError(MemoryLedger): + """A planted violation: a machinery error of a nested call leaks through the operation.""" + + def readings(self, envelope: CallEnvelope, name: str) -> tuple[Reading, ...]: + raise InterfaceContractError("CONTRACT_ENVELOPE_INVALID", "from a nested call", {}) + + class Seen(MemoryLedger): """Keeps every envelope it is called with.""" @@ -237,12 +313,15 @@ def test_a_declared_error_crosses_with_its_code_message_and_details(self) -> Non assert raised.value.__cause__ is None @pytest.mark.parametrize("serialized", [False, True]) - @pytest.mark.parametrize("implementation", [WrongCode, Crashing]) + @pytest.mark.parametrize( + "implementation", [WrongCode, Crashing, FakeRefusal, LeakedMachineryError] + ) def test_an_undeclared_error_is_refused_in_both_bindings( self, serialized: bool, implementation: type[MemoryLedger] ) -> None: with pytest.raises(InterfaceContractError) as raised: Bound(LEDGER, implementation(), serialized=serialized).readings(ENVELOPE, "a") + assert type(raised.value) is InterfaceContractError assert raised.value.code == "INTERFACE_ERROR_UNDECLARED" assert (raised.value.__cause__ is None) is serialized diff --git a/packages/contracts/tests/test_interface.py b/packages/contracts/tests/test_interface.py index 683f481c..ac58ab4d 100644 --- a/packages/contracts/tests/test_interface.py +++ b/packages/contracts/tests/test_interface.py @@ -11,6 +11,7 @@ from enum import Enum from typing import Any, Literal, Protocol +import pyarrow as pa import pytest from orvex.contracts.envelope import CallEnvelope from orvex.contracts.interface import InterfaceContract, Operation @@ -106,7 +107,7 @@ def test_the_description_names_every_part_of_the_contract(self) -> None: "name": "envelope", "kind": "positional-or-keyword", "type": f"{CallEnvelope.__module__}.CallEnvelope", - "default": False, + "default": None, } assert operations["record"] == { "capability": "write", @@ -119,7 +120,7 @@ def test_the_description_names_every_part_of_the_contract(self) -> None: "name": "reading", "kind": "positional-or-keyword", "type": READING_TYPE, - "default": False, + "default": None, }, ], "returns": "int", @@ -305,3 +306,47 @@ def test_replace_validates_again(self) -> None: assert code_of(lambda: dataclasses.replace(LEDGER, name="Bad")) == ( "INTERFACE_CONTRACT_INVALID" ) + + +@dataclass(frozen=True, slots=True) +class Use: + key: str | None + ttl_seconds: int | None + + +@dataclass(frozen=True, slots=True) +class Stored: + table: pa.Table + withheld: tuple[str, ...] + estimate: int | None + + +class ResultCache(Protocol): + def get(self, envelope: CallEnvelope, use: Use) -> Stored | None: ... + + def put(self, envelope: CallEnvelope, use: Use, stored: Stored) -> None: ... + + def size_bytes(self, envelope: CallEnvelope) -> int: ... + + def len(self, envelope: CallEnvelope) -> int: ... + + +def test_the_format_expresses_the_result_cache_that_g1_substitutes() -> None: + cache = InterfaceContract( + name="result-cache", + version=Version(1, 0), + protocol=ResultCache, + error_type=LedgerError, + capabilities=("cache",), + errors=(), + operations=( + Operation("get", "cache"), + Operation("put", "cache", changes_state=True), + Operation("size_bytes", "cache"), + Operation("len", "cache"), + ), + ) + assert cache.value_classes() == {CallEnvelope, Use, Stored} + assert cache.describe()["operations"]["get"]["returns"] == ( # type: ignore[index] + f"{Stored.__module__}.Stored | None" + ) diff --git a/packages/contracts/tests/test_kit.py b/packages/contracts/tests/test_kit.py index 78da7604..68159cad 100644 --- a/packages/contracts/tests/test_kit.py +++ b/packages/contracts/tests/test_kit.py @@ -210,6 +210,41 @@ def lax(*arguments: Any) -> ContractCallRefused | None: # noqa: ANN401 assert "reached ['names']" in message +class FakingV10(MemoryLedgerV10): + """A planted older provider that dodges a newer guarantee by faking the version refusal.""" + + def names(self, envelope: CallEnvelope) -> tuple[str, ...]: + if envelope.contract_version == "1.1": + raise ContractCallRefused("CONTRACT_VERSION_UNSUPPORTED", "faked", {}) + return super().names(envelope) + + +def newer_names_case(check: Callable[[KitClient], None]) -> Kit: + case = KitCase("names.newer", "names", "a newer guarantee on an old operation", check, V11) + return Kit(LEDGER, recorded=7, cases=(*LEDGER_KIT.cases, case)) + + +class TestFakedRefusals: + def test_a_refusal_faked_by_an_older_provider_is_a_failure(self) -> None: + kit = newer_names_case(_noop) + with pytest.raises(InterfaceContractError) as raised: + verify_kit(kit, FakingV10, provided=V10) + message = raised.value.message + assert "KIT_CASE_FAILED: names.newer (in process, consumer 1.1, provider 1.0)" in message + assert "INTERFACE_ERROR_UNDECLARED" in message + + def test_a_refusal_down_the_cause_chain_is_not_a_refusal(self) -> None: + def wrapped(ledger: KitClient) -> None: + try: + ledger.count() + except ContractCallRefused as error: + raise AssertionError("the case swallowed the refusal") from error + + kit = newer_names_case(wrapped) + report = run_kit(kit, MemoryLedgerV10, serialized=False, provided=V10) + assert next(r for r in report.results if r.id == "names.newer").outcome == "failed" + + class TestVersions: def test_consumers_are_n_1_n_and_the_newest(self) -> None: assert consumers(LEDGER, V11) == (V10, V11) @@ -241,12 +276,11 @@ def wrong(ledger: KitClient) -> None: if ledger.names() != ("z",): raise AssertionError("not z") - case = KitCase("names.z", "names", "a newer guarantee on an old operation", wrong, V11) - kit = Kit(LEDGER, recorded=7, cases=(*LEDGER_KIT.cases, case)) + kit = newer_names_case(wrong) with pytest.raises(InterfaceContractError) as raised: verify_kit(kit, MemoryLedgerV10, provided=V10) message = raised.value.message - assert "KIT_CASE_FAILED: names.z (in process, consumer 1.1, provider 1.0)" in message + assert "KIT_CASE_FAILED: names.newer (in process, consumer 1.1, provider 1.0)" in message def test_a_version_the_contract_does_not_describe_cannot_run(self) -> None: newer = Version(2, 0) diff --git a/packages/contracts/tests/test_manifest.py b/packages/contracts/tests/test_manifest.py index 5d986256..7a6e3ac0 100644 --- a/packages/contracts/tests/test_manifest.py +++ b/packages/contracts/tests/test_manifest.py @@ -45,7 +45,7 @@ def test_the_shipped_manifest_of_the_contracts_distribution_is_valid() -> None: assert manifest.module == "contracts" assert manifest.provides == () assert manifest.requires == () - assert manifest.runtime == ("pydantic",) + assert manifest.runtime == ("pyarrow", "pydantic") def test_provides_and_requires_may_be_absent() -> None: diff --git a/packages/server/src/orvex/server/assembly.py b/packages/server/src/orvex/server/assembly.py index db190e04..cd055642 100644 --- a/packages/server/src/orvex/server/assembly.py +++ b/packages/server/src/orvex/server/assembly.py @@ -1,17 +1,23 @@ """The assembly check (#486 G1): before the server composes the modules, every interface contract -a module requires has exactly one installed implementation, at a compatible version, with every -required capability and every operation of what it claims to provide. - -Every installed module may ship `module.toml` (`orvex.contracts.manifest`). An implementation -is loaded only from the module's `api` package, the one place this file imports a module by. -`UNSUPPORTED_OPERATION` cannot stand in for a capability: the contract format refuses to -declare it, and an operation the implementation lacks refuses the assembly. Any problem stops -the server before it opens a project or binds a port (exit status 3). +a module requires has one chosen implementation, at a compatible version, with every required +capability and every operation of what it claims to provide, each with the contract's exact +signature, envelope included. + +Every installed module may ship `module.toml` (`orvex.contracts.manifest`). When several +modules provide one contract, the configuration chooses one (`chosen`: contract name to module, +#486 §2.2); a chosen module outside the product's distributions, such as an alternative +implementation, is read by its module path. An implementation is loaded only from the module's +`api` package, the one place this file imports a module by. Its construction protocol: it is a +class, built from configuration by keyword arguments, so it takes no positional-only or +variadic positional parameter. `UNSUPPORTED_OPERATION` cannot stand in for a capability: the +contract format refuses to declare it, and an operation the implementation lacks refuses the +assembly. Any problem stops the server before it opens a project or binds a port (exit 3). """ from __future__ import annotations import importlib +import inspect from collections.abc import Mapping, Sequence from importlib.resources import files @@ -36,8 +42,10 @@ def assembly_problems( manifests: Sequence[Manifest], contracts: Mapping[str, InterfaceContract], implementations: Mapping[tuple[str, str], object], + chosen: Mapping[str, str] | None = None, ) -> list[str]: """Every reason the modules cannot be assembled; empty when they can.""" + choices = dict(chosen or {}) problems: list[str] = [] providers: dict[str, list[tuple[str, Provided]]] = {} for manifest in manifests: @@ -62,9 +70,20 @@ def assembly_problems( f"ASSEMBLY_NO_IMPLEMENTATION: {where}: no installed module provides it" ) continue + choice = choices.get(required.contract) + if choice is not None: + offered = [(module, entry) for module, entry in offered if module == choice] + if not offered: + problems.append( + f"ASSEMBLY_CHOICE_UNKNOWN: {where}: the chosen {choice} does not provide it" + ) + continue if len(offered) > 1: names = ", ".join(module for module, _ in offered) - problems.append(f"ASSEMBLY_AMBIGUOUS_IMPLEMENTATION: {where}: provided by {names}") + problems.append( + f"ASSEMBLY_AMBIGUOUS_IMPLEMENTATION: {where}: provided by {names}; " + "the configuration chooses one" + ) continue module, provided = offered[0] if not provided.version.provides(required.version): @@ -98,14 +117,44 @@ def _claim_problems( if unknown: return [f"ASSEMBLY_CAPABILITY_UNKNOWN: {where}: {unknown} are not its capabilities"] implementation = implementations.get((module, provided.contract)) - missing = [ - operation.name + if not isinstance(implementation, type): + return [f"ASSEMBLY_CONSTRUCTION_INVALID: {where}: {provided.implementation} is not a class"] + problems = _construction_problems(where, implementation) + operations = [ + operation for capability in provided.capabilities for operation in contract.operations_of(capability) - if not callable(getattr(implementation, operation.name, None)) + if provided.version.provides(contract.since(operation)) ] + missing = [o.name for o in operations if not callable(getattr(implementation, o.name, None))] if missing: - return [f"ASSEMBLY_OPERATION_MISSING: {where}: {provided.implementation} lacks {missing}"] + problems.append( + f"ASSEMBLY_OPERATION_MISSING: {where}: {provided.implementation} lacks {missing}" + ) + for operation in operations: + if operation.name not in missing: + mismatch = contract.mismatch(operation.name, implementation) + if mismatch is not None: + problems.append(f"ASSEMBLY_SIGNATURE_MISMATCH: {where}: {mismatch}") + return problems + + +def _construction_problems(where: str, implementation: type) -> list[str]: + """The construction protocol: built from configuration by keyword arguments only.""" + try: + parameters = inspect.signature(implementation).parameters.values() + except (TypeError, ValueError) as error: + return [f"ASSEMBLY_CONSTRUCTION_INVALID: {where}: {error}"] + positional = [ + parameter.name + for parameter in parameters + if parameter.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.VAR_POSITIONAL) + ] + if positional: + return [ + f"ASSEMBLY_CONSTRUCTION_INVALID: {where}: {implementation.__qualname__} takes " + f"{positional} by position; it is built from configuration by keyword" + ] return [] @@ -113,19 +162,36 @@ def check_assembly( manifests: Sequence[Manifest], contracts: Mapping[str, InterfaceContract], implementations: Mapping[tuple[str, str], object], + chosen: Mapping[str, str] | None = None, ) -> None: - problems = assembly_problems(manifests, contracts, implementations) + problems = assembly_problems(manifests, contracts, implementations, chosen) if problems: raise AssemblyRefused(problems) -def installed_manifests(identity: Identity) -> tuple[Manifest, ...]: - """The `module.toml` of every installed module that ships one.""" +def installed_manifests( + identity: Identity, chosen: Mapping[str, str] | None = None +) -> tuple[Manifest, ...]: + """The `module.toml` of every installed module that ships one, and of each chosen module.""" + modules = {module for module, _ in module_versions(identity)} | set((chosen or {}).values()) found = [] - for module, _ in module_versions(identity): - resource = files(f"{identity.python_namespace}.{module}").joinpath("module.toml") - if resource.is_file(): - found.append(parse_manifest(resource.read_text("utf-8"), module)) + problems = [] + for module in sorted(modules): + try: + resource = files(f"{identity.python_namespace}.{module}").joinpath("module.toml") + except ModuleNotFoundError: + problems.append(f"ASSEMBLY_CHOICE_UNKNOWN: the chosen module {module} is not installed") + continue + if not resource.is_file(): + continue + manifest = parse_manifest(resource.read_text("utf-8"), module) + if manifest.module != module: + problems.append( + f"ASSEMBLY_MANIFEST_MISPLACED: the manifest of {module} names {manifest.module}" + ) + found.append(manifest) + if problems: + raise AssemblyRefused(problems) return tuple(found) @@ -152,12 +218,13 @@ def load_implementations( return loaded -def check_installed_assembly(identity: Identity) -> None: - """The check run before the server starts, on the installed modules.""" - manifests = installed_manifests(identity) +def check_installed_assembly(identity: Identity, chosen: Mapping[str, str] | None = None) -> None: + """The check run before the server starts, on the installed modules and the choices.""" + manifests = installed_manifests(identity, chosen) contracts_package = importlib.import_module(f"{identity.python_namespace}.contracts") check_assembly( manifests, interface_contracts(contracts_package), load_implementations(identity.python_namespace, manifests), + chosen, ) diff --git a/packages/server/tests/test_assembly.py b/packages/server/tests/test_assembly.py index 0208ad8f..3ba6d91e 100644 --- a/packages/server/tests/test_assembly.py +++ b/packages/server/tests/test_assembly.py @@ -37,6 +37,8 @@ def put(self, envelope: CallEnvelope, key: str, value: bytes) -> None: ... def size(self, envelope: CallEnvelope) -> int: ... + def purge(self, envelope: CallEnvelope) -> None: ... + CACHE = InterfaceContract( name="cache", @@ -49,30 +51,76 @@ def size(self, envelope: CallEnvelope) -> int: ... Operation("get", "read"), Operation("size", "read"), Operation("put", "write", changes_state=True), + Operation("purge", "write", changes_state=True, since=Version(1, 1)), ), ) class Whole: - def get(self, key: str) -> bytes | None: + """A complete implementation at 1.1, with the contract's exact signatures.""" + + def get(self, envelope: CallEnvelope, key: str) -> bytes | None: return None - def put(self, key: str, value: bytes) -> None: + def put(self, envelope: CallEnvelope, key: str, value: bytes) -> None: return None - def size(self) -> int: + def size(self, envelope: CallEnvelope) -> int: + return 0 + + def purge(self, envelope: CallEnvelope) -> None: + return None + + +class AtOneZero: + """A complete implementation at 1.0: `purge` exists only from 1.1.""" + + def __init__(self, *, max_bytes: int = 1) -> None: + self.max_bytes = max_bytes + + def get(self, envelope: CallEnvelope, key: str) -> bytes | None: + return None + + def put(self, envelope: CallEnvelope, key: str, value: bytes) -> None: + return None + + def size(self, envelope: CallEnvelope) -> int: return 0 class WithoutPut: """A planted alternative that omits one operation.""" - def get(self, key: str) -> bytes | None: + def get(self, envelope: CallEnvelope, key: str) -> bytes | None: return None - def size(self) -> int: + def size(self, envelope: CallEnvelope) -> int: return 0 + def purge(self, envelope: CallEnvelope) -> None: + return None + + +class WithoutEnvelope(Whole): + """A planted alternative whose `get` drops the envelope.""" + + def get(self, key: str) -> bytes | None: # type: ignore[override] + return None + + +class Retyped(Whole): + """A planted alternative whose `size` answers a float.""" + + def size(self, envelope: CallEnvelope) -> float: # type: ignore[override] + return 0.0 + + +class Positional(Whole): + """A planted alternative that cannot be built from configuration by keyword.""" + + def __init__(self, path: str, /) -> None: + self.path = path + def provider( version: str = "1.1", capabilities: tuple[str, ...] = ("read", "write"), module: str = "alpha" @@ -86,13 +134,11 @@ def consumer(version: str = "1.0", capabilities: tuple[str, ...] = ("read", "wri return Manifest("server", (), (required,), (), ()) -def problems(*manifests: Manifest, implementation: object = None) -> list[str]: - chosen = { - (m.module, p.contract): Whole() if implementation is None else implementation - for m in manifests - for p in m.provides - } - return assembly_problems(manifests, {"cache": CACHE}, chosen) +def problems( + *manifests: Manifest, implementation: object = Whole, chosen: dict[str, str] | None = None +) -> list[str]: + loaded = {(m.module, p.contract): implementation for m in manifests for p in m.provides} + return assembly_problems(manifests, {"cache": CACHE}, loaded, chosen) def codes(found: list[str]) -> list[str]: @@ -102,7 +148,44 @@ def codes(found: list[str]) -> list[str]: def test_a_complete_implementation_at_a_compatible_version_assembles() -> None: assert problems(provider(), consumer()) == [] assert problems(provider("1.1"), consumer("1.1", ("read",))) == [] - check_assembly((provider(), consumer()), {"cache": CACHE}, {("alpha", "cache"): Whole()}) + check_assembly((provider(), consumer()), {"cache": CACHE}, {("alpha", "cache"): Whole}) + + +def test_an_implementation_needs_only_the_operations_of_its_version() -> None: + assert problems(provider("1.0"), consumer("1.0"), implementation=AtOneZero) == [] + found = problems(provider("1.1"), consumer("1.0"), implementation=AtOneZero) + assert codes(found) == ["ASSEMBLY_OPERATION_MISSING"] + assert "purge" in found[0] + + +@pytest.mark.parametrize( + ("implementation", "text"), + [ + (WithoutEnvelope, "get(key: str) -> bytes | None is not get(envelope: "), + (Retyped, "size(envelope: orvex.contracts.envelope.CallEnvelope) -> float is not"), + ], + ids=["envelope dropped", "result retyped"], +) +def test_a_planted_signature_mismatch_is_refused(implementation: type, text: str) -> None: + found = problems(provider(), consumer(), implementation=implementation) + assert codes(found) == ["ASSEMBLY_SIGNATURE_MISMATCH"] + assert text in found[0] + + +@pytest.mark.parametrize("implementation", [Whole(), Positional], ids=["instance", "positional"]) +def test_an_implementation_that_cannot_be_built_from_configuration_is_refused( + implementation: object, +) -> None: + found = problems(provider(), consumer(), implementation=implementation) + assert codes(found) == ["ASSEMBLY_CONSTRUCTION_INVALID"] + + +def test_the_configuration_chooses_one_of_several_implementations() -> None: + both = (provider(), provider(module="alternatives.cache_ipc"), consumer()) + assert codes(problems(*both)) == ["ASSEMBLY_AMBIGUOUS_IMPLEMENTATION"] + assert problems(*both, chosen={"cache": "alternatives.cache_ipc"}) == [] + assert problems(*both, chosen={"cache": "alpha"}) == [] + assert codes(problems(*both, chosen={"cache": "beta"})) == ["ASSEMBLY_CHOICE_UNKNOWN"] def test_a_planted_implementation_missing_a_required_capability_is_refused() -> None: @@ -112,13 +195,13 @@ def test_a_planted_implementation_missing_a_required_capability_is_refused() -> def test_a_planted_implementation_omitting_an_operation_is_refused() -> None: - found = problems(provider(), consumer(), implementation=WithoutPut()) + found = problems(provider(), consumer(), implementation=WithoutPut) assert codes(found) == ["ASSEMBLY_OPERATION_MISSING"] assert "put" in found[0] def test_an_operation_that_is_not_callable_is_refused() -> None: - hollow = types.SimpleNamespace(get=None, put=Whole().put, size=Whole().size) + hollow = type("Hollow", (Whole,), {"get": None}) assert codes(problems(provider(), consumer(), implementation=hollow)) == [ "ASSEMBLY_OPERATION_MISSING" ] @@ -158,13 +241,14 @@ def test_every_problem_is_reported_together() -> None: check_assembly( (provider(capabilities=("read",)), consumer("1.2")), {"cache": CACHE}, - {("alpha", "cache"): WithoutPut()}, + {("alpha", "cache"): WithoutPut}, ) assert raised.value.code == "ASSEMBLY_REFUSED" assert codes(raised.value.problems) == [ "ASSEMBLY_VERSION_INCOMPATIBLE", "ASSEMBLY_CAPABILITY_MISSING", ] + assert "ASSEMBLY_OPERATION_MISSING" not in codes(raised.value.problems) assert str(raised.value).startswith("ASSEMBLY_REFUSED: ") diff --git a/tests/tools/test_checks.py b/tests/tools/test_checks.py index eef9a594..53f85251 100644 --- a/tests/tools/test_checks.py +++ b/tests/tools/test_checks.py @@ -87,6 +87,18 @@ def test_from_namespace_import_module_is_reported(self) -> None: findings = boundary_findings(parse("from orvex import semantic\n"), "store", "orvex", "f") assert len(findings) == 1 + @pytest.mark.parametrize( + "source", + [ + 'import importlib\nimportlib.import_module("orvex.access.errors")\n', + 'from importlib import import_module\nimport_module("orvex.semantic")\n', + '__import__("orvex.transform.incremental")\n', + ], + ) + def test_a_constant_dynamic_import_is_reported(self, source: str) -> None: + findings = boundary_findings(parse(source), "store", "orvex", "f.py") + assert [f.code for f in findings] == ["MODULE_BOUNDARY_VIOLATION"] + def test_server_may_import_every_module(self) -> None: tree = parse("import orvex.semantic\nfrom orvex.store import x\n") assert boundary_findings(tree, "server", "orvex", "packages/server/src/a.py") == [] diff --git a/tests/tools/test_consumer_boundaries.py b/tests/tools/test_consumer_boundaries.py index ddfb7183..56d9416f 100644 --- a/tests/tools/test_consumer_boundaries.py +++ b/tests/tools/test_consumer_boundaries.py @@ -8,7 +8,7 @@ import pytest from tools.checks.boundaries import run -from tools.checks.consumers import consumer_findings, consumer_scan, consumer_summary +from tools.checks.consumers import AREAS, consumer_findings, consumer_scan, consumer_summary from tools.repository import REPOSITORY_ROOT, identity from .test_checks import make_repository @@ -108,8 +108,19 @@ def test_a_module_boundary_violation_still_fails(tmp_path: Path) -> None: assert run(make_repository(tmp_path / "r", files), empty, empty) == 1 -def test_the_repository_report_lists_what_remains() -> None: +# The files per area that still use module internals, measured on 2026-11-15 (#486 §1 counts +# 48 `server` source files; the `server` area here also holds its 32 test files). Report mode +# never lets a count grow, and each later step of #486 lowers it, to zero at step 7. +CEILINGS = {"server": 80, "runners": 10, "tools": 7, "tests": 66} + + +def test_the_repository_report_sees_the_coupling_and_never_more() -> None: found = consumer_scan(REPOSITORY_ROOT) - areas = consumer_summary(found) - assert all(f.code == "CONSUMER_BOUNDARY_VIOLATION" for f in found) - assert "server" in areas + files: dict[str, set[str]] = {area: set() for area in CEILINGS} + for finding in found: + area = next(a for a, start in AREAS.items() if finding.path.startswith(start)) + files[area].add(finding.path) + for area, ceiling in CEILINGS.items(): + assert 0 < len(files[area]) <= ceiling, (area, len(files[area])) + sources = {path for path in files["server"] if path.startswith("packages/server/src/")} + assert 0 < len(sources) <= 48 diff --git a/tests/tools/test_interface_compat.py b/tests/tools/test_interface_compat.py index 5c4ecf23..bedbb98e 100644 --- a/tests/tools/test_interface_compat.py +++ b/tests/tools/test_interface_compat.py @@ -100,6 +100,18 @@ def read(self, envelope: CallEnvelope, meter: str, at: int) -> int: ... def reset(self, envelope: CallEnvelope, meter: str) -> None: ... +class WithDefault(Protocol): + def read(self, envelope: CallEnvelope, meter: str = "main") -> int: ... + + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... + + +class OtherDefault(Protocol): + def read(self, envelope: CallEnvelope, meter: str = "spare") -> int: ... + + def reset(self, envelope: CallEnvelope, meter: str) -> None: ... + + class MoreOptional(Protocol): def read(self, envelope: CallEnvelope, meter: str, at: int | None = None) -> int: ... @@ -152,9 +164,37 @@ def test_a_removed_operation_fails(self) -> None: def test_a_renamed_error_code_fails(self) -> None: assert codes(incompatibilities(RELEASED, renamed_error())) == [ - "INTERFACE_ERROR_CODE_REMOVED" + "INTERFACE_OPERATION_ERRORS_CHANGED", + "INTERFACE_OPERATION_ERRORS_CHANGED", + "INTERFACE_ERROR_CODE_REMOVED", ] + @pytest.mark.parametrize( + ("read_errors", "text"), + [ + (("METER_UNKNOWN", "METER_LOCKED"), "raises ['METER_LOCKED'] more and [] fewer"), + ((), "raises [] more and ['METER_UNKNOWN'] fewer"), + ], + ids=["added", "removed"], + ) + def test_an_error_code_added_to_or_removed_from_an_operation_fails( + self, read_errors: tuple[str, ...], text: str + ) -> None: + changed = contract( + operations=( + Operation("read", "read", read_errors), + Operation("reset", "write", ("METER_UNKNOWN", "METER_LOCKED")), + ) + ) + found = incompatibilities(RELEASED, recorded(changed)) + assert codes(found) == ["INTERFACE_OPERATION_ERRORS_CHANGED"] + assert f"meter.read {text}" in found[0] + + def test_a_changed_default_value_fails(self) -> None: + released = recorded(contract(protocol=WithDefault)) + changed = recorded(contract(protocol=OtherDefault)) + assert codes(incompatibilities(released, changed)) == ["INTERFACE_SIGNATURE_CHANGED"] + def test_a_changed_kit_expectation_fails(self) -> None: assert codes(incompatibilities(RELEASED, changed_expectation())) == [ "INTERFACE_KIT_EXPECTATION_CHANGED" diff --git a/tools/checks/boundaries.py b/tools/checks/boundaries.py index ba1d3d61..4bb2b1e7 100644 --- a/tools/checks/boundaries.py +++ b/tools/checks/boundaries.py @@ -2,7 +2,8 @@ with the materialization and data-reader checks. 1. MODULE_BOUNDARY_VIOLATION (here): code in `packages//` imports only its own module, - `.contracts`, the standard library and third-party packages; `server` composes all. + `.contracts`, the standard library and third-party packages, statically or by a + constant dynamic import (`tools/checks/imports.py`); `server` composes all. 2. MATERIALIZATION_IN_PYTHON: `tools/checks/materialization.py`. 3. DATA_READ_OUTSIDE_DUCKDB: `tools/checks/data_readers.py`. 4. CONSUMER_BOUNDARY_VIOLATION: `tools/checks/consumers.py`, reported without failing until @@ -24,6 +25,7 @@ reader_scan, stale_reader_entries, ) +from tools.checks.imports import imported_names from tools.checks.materialization import ( ALLOW_LIST, MATERIALIZING_NAMES, @@ -38,7 +40,6 @@ "AllowListError", "Finding", "boundary_findings", - "imported_modules", "load_allow_list", "load_reader_allow_list", "main", @@ -56,23 +57,12 @@ COMPOSER = "server" -def imported_modules(tree: ast.AST) -> list[tuple[str, int]]: - found: list[tuple[str, int]] = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - found.extend((alias.name, node.lineno) for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - found.append((node.module, node.lineno)) - found.extend((f"{node.module}.{alias.name}", node.lineno) for alias in node.names) - return found - - def boundary_findings(tree: ast.AST, owner: str, namespace: str, relative: str) -> list[Finding]: if owner == COMPOSER: return [] findings: list[Finding] = [] reported: set[tuple[str, int]] = set() - for name, line in imported_modules(tree): + for name, line in imported_names(tree): parts = name.split(".") if len(parts) < 2 or parts[0] != namespace or parts[1] not in MODULES: continue diff --git a/tools/checks/consumers.py b/tools/checks/consumers.py index 10ddfb35..9fc9293c 100644 --- a/tools/checks/consumers.py +++ b/tools/checks/consumers.py @@ -2,8 +2,8 @@ tests import only `.contracts`; `server/assembly.py` may also import the modules' `api` packages, and the server's own files the server. -CONSUMER_BOUNDARY_VIOLATION: any other import of a module, written as `import`, `from … import` -or a constant `importlib.import_module(…)` / `__import__(…)`. In report mode until #486 step 7: +CONSUMER_BOUNDARY_VIOLATION: any other import of a module, as `tools/checks/imports.py` reads +it (static, or dynamic with a constant name). In report mode until #486 step 7: `python -m tools.checks.boundaries` prints every violation and a summary per area, and fails only on the module boundaries. """ @@ -15,6 +15,7 @@ from pathlib import Path from tools.checks.architecture import Finding +from tools.checks.imports import imported_names from tools.repository import MODULES, identity CODE = "CONSUMER_BOUNDARY_VIOLATION" @@ -27,34 +28,12 @@ } SHARED = "contracts" PUBLIC = "api" -DYNAMIC = {"import_module", "__import__"} def assembly(namespace: str) -> str: return f"packages/server/src/{namespace}/server/assembly.py" -def _imports(tree: ast.AST) -> list[tuple[str, int]]: - found: list[tuple[str, int]] = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - found.extend((alias.name, node.lineno) for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - found.append((node.module, node.lineno)) - found.extend((f"{node.module}.{alias.name}", node.lineno) for alias in node.names) - elif isinstance(node, ast.Call) and node.args: - function = node.func - name = ( - function.attr - if isinstance(function, ast.Attribute) - else getattr(function, "id", "") - ) - first = node.args[0] - if name in DYNAMIC and isinstance(first, ast.Constant) and isinstance(first.value, str): - found.append((first.value, node.lineno)) - return found - - def _area(relative: str) -> str | None: return next((area for area, start in AREAS.items() if relative.startswith(start)), None) @@ -66,7 +45,7 @@ def consumer_findings(tree: ast.AST, namespace: str, relative: str) -> list[Find return [] findings: list[Finding] = [] reported: set[tuple[str, int]] = set() - for name, line in _imports(tree): + for name, line in imported_names(tree): parts = name.split(".") if len(parts) < 2 or parts[0] != namespace or parts[1] not in MODULES: continue diff --git a/tools/checks/imports.py b/tools/checks/imports.py new file mode 100644 index 00000000..b892570e --- /dev/null +++ b/tools/checks/imports.py @@ -0,0 +1,42 @@ +"""The absolute imports a Python file makes, as the boundary rules read them: `import`, +`from … import`, and `importlib.import_module` or `__import__` called with a constant name. + +Used by the module rule (`boundaries.py`) and the consumer rule (`consumers.py`), and the +dynamic-call test by the import-cycle note (`structure.py`). `tools/checks/module_dependencies.py` +keeps its own small reader on purpose: it runs with `python -I` in a clean environment outside +the repository (#486 G4), where the `tools` package is not importable. +""" + +from __future__ import annotations + +import ast +from typing import TypeGuard + +DYNAMIC = frozenset({"import_module", "__import__"}) + + +def dynamic_import(node: ast.AST) -> TypeGuard[ast.Call]: + """Whether `node` calls `import_module` or `__import__`, by name or as an attribute.""" + if not isinstance(node, ast.Call): + return False + function = node.func + if isinstance(function, ast.Name): + return function.id in DYNAMIC + return isinstance(function, ast.Attribute) and function.attr in DYNAMIC + + +def imported_names(tree: ast.AST) -> list[tuple[str, int]]: + """Each absolute name imported, with its line: `a.b` for `import a.b`, both `a` and `a.b` + for `from a import b`, and the constant first argument of a dynamic import.""" + found: list[tuple[str, int]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.extend((alias.name, node.lineno) for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.append((node.module, node.lineno)) + found.extend((f"{node.module}.{alias.name}", node.lineno) for alias in node.names) + elif dynamic_import(node) and node.args: + first = node.args[0] + if isinstance(first, ast.Constant) and isinstance(first.value, str): + found.append((first.value, node.lineno)) + return found diff --git a/tools/checks/module_dependencies.py b/tools/checks/module_dependencies.py index 396670a2..58ac306c 100644 --- a/tools/checks/module_dependencies.py +++ b/tools/checks/module_dependencies.py @@ -1,11 +1,15 @@ -"""Inside a module's clean environment (#486 G4): the distributions its code imports equal those -it declares, and its own tests import no repository helper. +"""The dependency check of one standalone module (#486 G4): in its clean environment, what the +module imports equals what it declares, its tests use no repository helper, and every contract +it requires has a public test adapter. /bin/python -I tools/checks/module_dependencies.py NAMESPACE DISTRIBUTION TESTS -Run by `tools/ci/standalone.py` with the environment's interpreter, so it uses the standard -library and the installed contracts package only. MODULE_DEPENDENCY_UNDECLARED: the code imports -a distribution it does not declare (or one nothing installed provides); +Run by `tools/ci/standalone.py` with the environment's interpreter (`python -I`), so it uses +the standard library and the installed contracts package only; for that reason it reads imports +with its own small reader instead of `tools/checks/imports.py`, which is not installed there. + +MODULE_DEPENDENCY_UNDECLARED: the code imports a distribution it does not declare (or one +nothing installed provides); MODULE_DEPENDENCY_UNUSED: it declares one it never imports; MODULE_MANIFEST_DEPENDENCIES_DIFFER: `module.toml` and the distribution disagree; MODULE_MANIFEST_MISSING: no `module.toml`; MODULE_TEST_IMPORTS_REPOSITORY: a test imports `tests`, `conformance` or `tools`; diff --git a/tools/checks/structure.py b/tools/checks/structure.py index 7cbf42ba..29eba8d2 100644 --- a/tools/checks/structure.py +++ b/tools/checks/structure.py @@ -30,6 +30,7 @@ from dataclasses import dataclass from pathlib import Path +from tools.checks.imports import dynamic_import from tools.repository import REPOSITORY_ROOT LIMIT = 800 @@ -315,13 +316,7 @@ def dynamic(root: Path) -> list[Finding]: found = [] for _, path in _sources(root): for node in ast.walk(ast.parse(path.read_text("utf-8"))): - if isinstance(node, ast.Call) and ( - ( - isinstance(node.func, ast.Name) - and node.func.id in {"__import__", "import_module"} - ) - or (isinstance(node.func, ast.Attribute) and node.func.attr == "import_module") - ): + if dynamic_import(node): found.append( Finding( "note", diff --git a/tools/contracts/interface_compat.py b/tools/contracts/interface_compat.py index 472239c8..302186ea 100644 --- a/tools/contracts/interface_compat.py +++ b/tools/contracts/interface_compat.py @@ -6,15 +6,17 @@ (the newest `v` tag HEAD contains): within a major version a contract only grows. These fail, unless the major version went up: a removed contract (INTERFACE_CONTRACT_REMOVED), a lowered version (INTERFACE_VERSION_LOWERED), a removed operation (INTERFACE_OPERATION_REMOVED), -a changed signature, capability, state change or version of an operation -(INTERFACE_SIGNATURE_CHANGED), a removed capability (INTERFACE_CAPABILITY_REMOVED), a removed -or renamed error code (INTERFACE_ERROR_CODE_REMOVED), a removed kit case +a changed signature (a default value included), capability, state change or version of an +operation (INTERFACE_SIGNATURE_CHANGED), an error code added to or removed from an operation +(INTERFACE_OPERATION_ERRORS_CHANGED), a removed capability (INTERFACE_CAPABILITY_REMOVED), a +removed or renamed error code (INTERFACE_ERROR_CODE_REMOVED), a removed kit case (INTERFACE_KIT_CASE_REMOVED), a changed expectation (INTERFACE_KIT_EXPECTATION_CHANGED), and an operation or kit case added since the release that claims to exist at the released version -(INTERFACE_ADDITION_UNVERSIONED). Allowed: new contracts, and new operations, capabilities, -error codes, kit cases and parameters with a default at a new minor version. No release tag -fails closed (INTERFACE_RELEASE_UNKNOWN); a release without a record released no interface -contract. Running N against N-1 in both directions is the kit's (`orvex.contracts.kit_run`). +(INTERFACE_ADDITION_UNVERSIONED). Allowed at a new minor version: new contracts, operations, +capabilities, kit cases, the error codes of new operations and parameters with a default. No +release tag fails closed (INTERFACE_RELEASE_UNKNOWN); a release without a record released no +interface contract. Running N against N-1 in both directions is the kit's +(`orvex.contracts.kit_run`). """ from __future__ import annotations @@ -58,16 +60,22 @@ def _operation_problems( changed = ( len(after) < len(before) or [_shape(p) for p in after[: len(before)]] != [_shape(p) for p in before] - or any(p["default"] and not q["default"] for p, q in zip(before, after, strict=False)) - or any(not p["default"] for p in after[len(before) :]) + or any(p["default"] is None for p in after[len(before) :]) ) if changed: problems.append(f"INTERFACE_SIGNATURE_CHANGED: {where} changed its parameters") + added = sorted(set(new["errors"]) - set(old["errors"])) + removed = sorted(set(old["errors"]) - set(new["errors"])) + if added or removed: + problems.append( + f"INTERFACE_OPERATION_ERRORS_CHANGED: {where} raises {added} more and {removed} fewer" + ) return problems -def _shape(parameter: dict[str, Any]) -> tuple[str, str, str]: - return parameter["name"], parameter["kind"], parameter["type"] +def _shape(parameter: dict[str, Any]) -> tuple[str, str, str, str | None]: + """Everything a parameter promises its callers: name, kind, type and default value.""" + return parameter["name"], parameter["kind"], parameter["type"], parameter["default"] def _unversioned(name: str, old: dict[str, Any], new: dict[str, Any]) -> list[str]: From ffad664298b87196286ab885158008b919cb7a28 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 08:11:24 -0400 Subject: [PATCH 08/21] fix(store): watch signals with a selector, so 1,024 open files never break it select.select refuses a descriptor at or past FD_SETSIZE; CI's test part 9 reached it and the ctrl-c watcher thread raised. The scheduler's watcher had the same defect: both now use a selector, and each has a regression test with 1,100 open files. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- packages/server/src/orvex/server/scheduler.py | 26 ++++++------ .../server/tests/test_scheduler_signals.py | 41 +++++++++++++++++++ packages/store/src/orvex/store/ctrl_c.py | 24 ++++++----- packages/store/tests/test_ctrl_c.py | 37 +++++++++++++++++ 4 files changed, 105 insertions(+), 23 deletions(-) create mode 100644 packages/server/tests/test_scheduler_signals.py create mode 100644 packages/store/tests/test_ctrl_c.py diff --git a/packages/server/src/orvex/server/scheduler.py b/packages/server/src/orvex/server/scheduler.py index 549778e2..0319c078 100644 --- a/packages/server/src/orvex/server/scheduler.py +++ b/packages/server/src/orvex/server/scheduler.py @@ -27,7 +27,7 @@ import datetime as dt import hashlib import os -import select +import selectors import signal import socket import threading @@ -437,17 +437,19 @@ def _watch_signals(stop: Callable[[int, FrameType | None], None]) -> Callable[[] earlier = signal.set_wakeup_fd(sender.fileno(), warn_on_full_buffer=False) def watch() -> None: - while not finished.is_set(): - ready, _, _ = select.select([receiver], [], [], 0.2) - if not ready: - continue - with contextlib.suppress(OSError): - received = receiver.recv(256) - if earlier != -1: - os.write(earlier, received) - for number in numbers: - if number in received: - stop(number, None) + # A selector, not `select.select`: a process with 1,024 open files or more is fine. + with selectors.DefaultSelector() as selector: + selector.register(receiver, selectors.EVENT_READ) + while not finished.is_set(): + if not selector.select(0.2): + continue + with contextlib.suppress(OSError): + received = receiver.recv(256) + if earlier != -1: + os.write(earlier, received) + for number in numbers: + if number in received: + stop(number, None) watcher = threading.Thread(target=watch, name="schedule-stop", daemon=True) watcher.start() diff --git a/packages/server/tests/test_scheduler_signals.py b/packages/server/tests/test_scheduler_signals.py new file mode 100644 index 00000000..966ea8f7 --- /dev/null +++ b/packages/server/tests/test_scheduler_signals.py @@ -0,0 +1,41 @@ +"""The scheduler's stop signals are watched even when the process holds 1,024 open files or +more.""" + +from __future__ import annotations + +import os +import signal +import time +from collections.abc import Iterator +from types import FrameType + +import pytest +from orvex.server.scheduler import _watch_signals + + +@pytest.fixture +def many_files() -> Iterator[None]: + """Enough open descriptors that the next one is past `select`'s limit of 1,024.""" + read, write = os.pipe() + opened = [os.dup(read) for _ in range(1100)] + try: + yield + finally: + for descriptor in (*opened, read, write): + os.close(descriptor) + + +@pytest.mark.usefixtures("many_files") +def test_a_stop_signal_is_seen_with_many_open_files() -> None: + stopped: list[int] = [] + + def stop(number: int, frame: FrameType | None) -> None: + stopped.append(number) + + restore = _watch_signals(stop) + try: + signal.raise_signal(signal.SIGTERM) + time.sleep(0.5) # the watcher thread reads the wakeup byte + finally: + restore() + assert signal.SIGTERM in stopped diff --git a/packages/store/src/orvex/store/ctrl_c.py b/packages/store/src/orvex/store/ctrl_c.py index 59178ba9..b73e2b5b 100644 --- a/packages/store/src/orvex/store/ctrl_c.py +++ b/packages/store/src/orvex/store/ctrl_c.py @@ -6,7 +6,7 @@ import contextlib import os -import select +import selectors import signal import socket import threading @@ -56,16 +56,18 @@ def handler(number: int, frame: FrameType | None) -> None: return lambda: None def watch() -> None: - while not stop.is_set(): - ready, _, _ = select.select([receiver], [], [], 0.2) - if not ready: - continue - with contextlib.suppress(OSError): - received = receiver.recv(256) - if earlier != -1: - os.write(earlier, received) # whoever watched signals before still sees them - if signal.SIGINT in received: - interrupt() + # A selector, not `select.select`: a process with 1,024 open files or more is fine. + with selectors.DefaultSelector() as selector: + selector.register(receiver, selectors.EVENT_READ) + while not stop.is_set(): + if not selector.select(0.2): + continue + with contextlib.suppress(OSError): + received = receiver.recv(256) + if earlier != -1: # whoever watched signals before still sees them + os.write(earlier, received) + if signal.SIGINT in received: + interrupt() watcher = threading.Thread(target=watch, name="load-ctrl-c", daemon=True) watcher.start() diff --git a/packages/store/tests/test_ctrl_c.py b/packages/store/tests/test_ctrl_c.py new file mode 100644 index 00000000..6ceca5e7 --- /dev/null +++ b/packages/store/tests/test_ctrl_c.py @@ -0,0 +1,37 @@ +"""Ctrl-C during a load is watched even when the process holds 1,024 open files or more.""" + +from __future__ import annotations + +import os +import signal +import time +from collections.abc import Iterator + +import pytest +from orvex.store.ctrl_c import _on_ctrl_c + + +@pytest.fixture +def many_files() -> Iterator[None]: + """Enough open descriptors that the next one is past `select`'s limit of 1,024.""" + read, write = os.pipe() + opened = [os.dup(read) for _ in range(1100)] + try: + yield + finally: + for descriptor in (*opened, read, write): + os.close(descriptor) + + +@pytest.mark.usefixtures("many_files") +def test_ctrl_c_calls_back_once_with_many_open_files() -> None: + called: list[bool] = [] + pressed: list[bool] = [] + restore = _on_ctrl_c(lambda: called.append(True), pressed) + try: + signal.raise_signal(signal.SIGINT) + time.sleep(0.5) # the watcher thread reads the wakeup byte + finally: + restore() + assert called == [True] + assert pressed # noted by the handler and by the watcher From 91bb4c2685a757870c077c429087ec9792c238a3 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 08:19:21 -0400 Subject: [PATCH 09/21] feat(contracts): aware datetimes cross exactly, and envelopes come from one call context - AwareDatetime and constrained types are value types; IANA zones, datetime.timezone and pydantic's TzInfo each come back as themselves; a naive datetime is refused; every generated store type may now cross (pydantic-core declared); - CallContext holds the correlation id, actor, project and revision, refuses missing or placeholder values, and makes every envelope; the server builds one correlation id per HTTP request (a well-formed X-Request-Id is kept), CLI command or scheduled run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- ...-interface-contracts-kits-and-manifests.md | 4 +- docs/process/git-workflow.md | 1 + packages/contracts/README.md | 2 +- packages/contracts/pyproject.toml | 2 +- .../contracts/src/orvex/contracts/codec.py | 53 +++++++++---- .../contracts/src/orvex/contracts/context.py | 65 +++++++++++++++ .../contracts/src/orvex/contracts/envelope.py | 45 ++++++++--- .../src/orvex/contracts/interface_errors.py | 3 +- .../src/orvex/contracts/kit_client.py | 25 +++--- .../contracts/src/orvex/contracts/kit_run.py | 9 ++- .../contracts/src/orvex/contracts/module.toml | 5 +- .../src/orvex/contracts/serializable.py | 21 +++-- packages/contracts/tests/test_binding.py | 33 +++++++- packages/contracts/tests/test_context.py | 63 +++++++++++++++ packages/contracts/tests/test_envelope.py | 6 +- packages/contracts/tests/test_interface.py | 19 ++++- packages/contracts/tests/test_kit.py | 2 +- packages/contracts/tests/test_manifest.py | 2 +- packages/server/src/orvex/server/app.py | 5 +- .../server/src/orvex/server/call_context.py | 42 ++++++++++ .../server/src/orvex/server/server_app.py | 5 +- packages/server/tests/test_call_context.py | 79 +++++++++++++++++++ 22 files changed, 421 insertions(+), 70 deletions(-) create mode 100644 packages/contracts/src/orvex/contracts/context.py create mode 100644 packages/contracts/tests/test_context.py create mode 100644 packages/server/src/orvex/server/call_context.py create mode 100644 packages/server/tests/test_call_context.py diff --git a/docs/adr/0033-interface-contracts-kits-and-manifests.md b/docs/adr/0033-interface-contracts-kits-and-manifests.md index c4c2cdbc..1c284699 100644 --- a/docs/adr/0033-interface-contracts-kits-and-manifests.md +++ b/docs/adr/0033-interface-contracts-kits-and-manifests.md @@ -6,8 +6,8 @@ ADR-0004 makes modules talk only through contracts, but its contracts are data contracts: they say what crosses a boundary, never which operations a module offers. On 2026-11-14 no module imported another module's internals, yet `server`, the conformance runners, `tools/` and the repository tests used 65 internal paths of the modules (#486 §1), the boundaries check exempted `server` as the composition root, and no module had ever been installed or tested outside the repository. Nothing proved that another implementation of a module could replace it. ## Decision -1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version (`orvex.contracts.version`), groups the operations into capabilities, lists the error codes each may raise through the contract's error type, marks the operations that change state and the version each exists from. Only values cross: the annotated inputs and outputs are plain values, Arrow tables, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. The serializing codec keeps each exactly: a model field by field (its own JSON would turn an IANA zone into an offset), an Arrow table as Arrow IPC. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`, its kit in `orvex.contracts..kit` and its public test adapter, when it has one, in `orvex.contracts..adapter` (`orvex.contracts.registry`). -2. **The call envelope.** Every operation takes a `CallEnvelope` (`orvex.contracts.envelope`) as its first argument: `contract_name` and `contract_version` (the contract's version, never the module's or a physical format's), `correlation_id`, the effective `actor` or a verifiable reference to it, `project_id` and `revision`, and an `idempotency_key`, required on every operation that changes state. Every binding (`orvex.contracts.binding`) validates it before the implementation runs and fails closed with a typed `ContractCallRefused`: CONTRACT_ENVELOPE_INVALID for a missing, blank or mistyped field, a missing key or another contract's envelope; CONTRACT_VERSION_UNSUPPORTED for another major version or an operation that the provider's or the caller's version does not have. A caller at a newer minor version is served for the operations the provider has. +1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version (`orvex.contracts.version`), groups the operations into capabilities, lists the error codes each may raise through the contract's error type, marks the operations that change state and the version each exists from. Only values cross: the annotated inputs and outputs are plain values, Arrow tables, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. The serializing codec keeps each exactly: a model field by field (its own JSON would turn an IANA zone into an offset), an Arrow table as Arrow IPC. Datetimes are aware: pydantic's `AwareDatetime` is a value type, an IANA zone and a fixed offset (`datetime.timezone`, or the `pydantic_core.TzInfo` pydantic parses an offset into) each come back as themselves, and a naive datetime is refused. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`, its kit in `orvex.contracts..kit` and its public test adapter, when it has one, in `orvex.contracts..adapter` (`orvex.contracts.registry`). +2. **The call envelope.** Every operation takes a `CallEnvelope` (`orvex.contracts.envelope`) as its first argument: `contract_name` and `contract_version` (the contract's version, never the module's or a physical format's), `correlation_id`, the effective `actor` or a verifiable reference to it, `project_id` and `revision`, and an `idempotency_key`, required on every operation that changes state. Every binding (`orvex.contracts.binding`) validates it before the implementation runs and fails closed with a typed `ContractCallRefused`: CONTRACT_ENVELOPE_INVALID for a missing, blank or mistyped field, a missing key or another contract's envelope; CONTRACT_VERSION_UNSUPPORTED for another major version or an operation that the provider's or the caller's version does not have. A caller at a newer minor version is served for the operations the provider has. A placeholder (`unknown`, `n/a`, `none` and the like, `envelope.PLACEHOLDERS`) is refused like a missing value. A consumer never writes an envelope by hand: it holds a `CallContext` (`orvex.contracts.context`: `correlation_id`, actor, `project_id`, revision, refused with CONTRACT_CONTEXT_INVALID when any is missing or a placeholder), built once at each server entry point (`orvex.server.call_context`: an HTTP request keeps a well-formed `X-Request-Id` as its correlation id or gets a new one, and a CLI command or scheduled run gets a new one), and every consumer client takes that context per call and makes the envelope from it (`CallContext.envelope`). The entry points build it where they first make a contract call, as each consumer moves onto its contract (#486 steps 2-7). 3. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise, their expectation in words and the version they exist from, a recorded case count and no skip list. A kit leaves no operation without a case, and each operation that changes state has an idempotency case: the call is sent twice with one key and its effect must happen exactly once. The runner (`orvex.contracts.kit_run`, `kit_client`, `codec`) runs every case on a fresh implementation, once in process and once through the serializing binding, by a consumer at N-1 and at N, and adds the envelope cases the contract implies (each missing field and key, another contract, another major version), which must be refused before the implementation is reached. A provider at N-1 runs the kit of N: the cases of its version pass, the newer ones pass or are refused with CONTRACT_VERSION_UNSUPPORTED, never answered wrongly. An implementation that passes its contract's whole kit may replace the module. 4. **Consumers know contracts, not modules** (amends rule 3). `server` stays the only composition root, but only `server/assembly.py` names implementations, and only through each module's public `api` package. Every other consumer (`server`, the conformance runners, `tools/`, the repository tests) imports only `orvex.contracts`. `tools/checks/consumers.py` enforces it, in report mode until #486 step 7 makes it blocking and removes the `server` exemption. 5. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` class that implements each) and requires (version, capabilities), and the distributions its code and its tests import. A module is named by its package path under the namespace, so an alternative implementation (`alternatives.result_cache_ipc`) has a manifest too. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation, or several and the configuration chose none (or chose one that does not provide it); a version that does not provide the required one; a missing capability; an implementation that is not a class built from configuration by keyword arguments, lacks an operation of what it claims at its version, or whose method differs from the contract's signature, envelope included. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. `module.toml` does not replace `contracts.toml`: `contracts.toml` declares the language-neutral data contracts a module produces and consumes, by their `vN` versions (ADR-0004 rule 6, `versioning.md` §3), and the server checks them as before; `module.toml` declares the interface contracts, by `MAJOR.MINOR`, and the module's dependencies. Merging them would change the checked data-contract declarations of every module at once, which is outside #486. diff --git a/docs/process/git-workflow.md b/docs/process/git-workflow.md index 26aa0e9e..cf9acc4e 100644 --- a/docs/process/git-workflow.md +++ b/docs/process/git-workflow.md @@ -125,6 +125,7 @@ packages/contracts/src/orvex/contracts/ generated contract types !packages/contracts/src/orvex/contracts/registry.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/serializable.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/adapter.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/context.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/envelope.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/kit_client.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/version.py hand-written: counted (#486, interface contracts) diff --git a/packages/contracts/README.md b/packages/contracts/README.md index f3c7f70c..2f6e07b7 100644 --- a/packages/contracts/README.md +++ b/packages/contracts/README.md @@ -3,7 +3,7 @@ Every module may import this package. It holds: - Python types generated from [`contracts/`](../../contracts/README.md) by `python -m tools.contracts.generate` (`/v.py`), never edited by hand; -- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, state changes) and `version.py`; `envelope.py` (the call envelope every binding validates); `kit.py`, `kit_client.py` and `kit_run.py` (conformance kits: cases, recorded count, idempotency, no skips, run in process and serialized by consumers at N-1 and N); `binding.py`, `codec.py` and `serializable.py` (what may cross a contract); `adapter.py` (public test adapters); `manifest.py` (`module.toml`); `registry.py` and `interface_errors.py`; +- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, state changes) and `version.py`; `envelope.py` (the call envelope every binding validates) and `context.py` (the call context envelopes are made from); `kit.py`, `kit_client.py` and `kit_run.py` (conformance kits: cases, recorded count, idempotency, no skips, run in process and serialized by consumers at N-1 and N); `binding.py`, `codec.py` and `serializable.py` (what may cross a contract); `adapter.py` (public test adapters); `manifest.py` (`module.toml`); `registry.py` and `interface_errors.py`; - `module.toml`, this module's manifest, and `interface-record.json` (beside `pyproject.toml`), the record of every interface contract that `python -m tools.contracts.interface_record` writes and `python -m tools.contracts.interface_compat` compares with the last release. Module pack: [`docs/modules/contracts.md`](../../docs/modules/contracts.md). diff --git a/packages/contracts/pyproject.toml b/packages/contracts/pyproject.toml index 67a51820..14ee8049 100644 --- a/packages/contracts/pyproject.toml +++ b/packages/contracts/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" license = "Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.14" -dependencies = ["pyarrow==26.0.1", "pydantic==2.14.3"] +dependencies = ["pyarrow==26.0.1", "pydantic==2.14.3", "pydantic-core==2.50.4"] [tool.setuptools.packages.find] where = ["src"] diff --git a/packages/contracts/src/orvex/contracts/codec.py b/packages/contracts/src/orvex/contracts/codec.py index 49d87189..cb21a0b0 100644 --- a/packages/contracts/src/orvex/contracts/codec.py +++ b/packages/contracts/src/orvex/contracts/codec.py @@ -2,12 +2,14 @@ Every value becomes tagged JSON: exact types survive (a tuple stays a tuple, `bool` is never an `int`), a `Decimal` keeps its scale, an integer of any width stays exact, a `datetime` keeps its -zone (an IANA zone by its key) and its `fold`. A pydantic model crosses field by field through -the same tree and is validated again when rebuilt, so nothing inside it is weakened (its own -JSON would turn an IANA zone into a fixed offset). An Arrow table crosses as Arrow IPC, which -keeps every type, NULL and time zone of its schema. Only the classes the contract declares -(`InterfaceContract.value_classes`) are built back, and only of exactly that class. Anything -else is refused with INTERFACE_VALUE_NOT_SERIALIZABLE: a live object never crosses. +zone (an IANA zone by its key, or a fixed offset of `datetime.timezone` or pydantic's `TzInfo`, +each rebuilt as itself) and its `fold`; a naive datetime is refused. A pydantic model crosses +field by field through the same tree and is validated again when rebuilt, so nothing inside it +is weakened (its own JSON would turn an IANA zone into a fixed offset). An Arrow table crosses +as Arrow IPC, which keeps every type, NULL and time zone of its schema. Only the classes the +contract declares (`InterfaceContract.value_classes`) are built back, and only of exactly that +class. Anything else is refused with INTERFACE_VALUE_NOT_SERIALIZABLE: a live object never +crosses. """ from __future__ import annotations @@ -15,18 +17,21 @@ import base64 import dataclasses import json -from datetime import date, datetime, time, timedelta, timezone +from datetime import date, datetime, time, timedelta, timezone, tzinfo from decimal import Decimal from enum import Enum from typing import Any from zoneinfo import ZoneInfo import pyarrow as pa -from pydantic import BaseModel +from pydantic import BaseModel, RootModel +from pydantic_core import TzInfo from .interface_errors import InterfaceContractError CODE = "INTERFACE_VALUE_NOT_SERIALIZABLE" +# The fixed-offset zones that cross, by the name their kind is recorded with. +OFFSET_KINDS: dict[type, str] = {timezone: "timezone", TzInfo: "pydantic"} def qualified(kind: type) -> str: @@ -142,6 +147,8 @@ def _tagged(self, tag: str, payload: Any, tree: dict[str, Any]) -> object: # no return kind[tree["member"]] if tag == "$model" and issubclass(kind, BaseModel): fields = {name: self._value(item) for name, item in tree["fields"].items()} + if issubclass(kind, RootModel): + return kind.model_validate(fields["root"]) return kind.model_validate(fields, by_alias=False, by_name=True) if tag == "$dataclass" and dataclasses.is_dataclass(kind): return kind(**{name: self._value(item) for name, item in tree["fields"].items()}) @@ -156,22 +163,34 @@ def _ipc(table: pa.Table) -> bytes: def _zone(value: datetime) -> dict[str, object]: + """The zone of an aware datetime: an IANA zone by its key, a fixed offset with the kind of + object that holds it (`datetime.timezone`, or the `TzInfo` pydantic parses an offset into). + A naive datetime never crosses: time zones are explicit.""" zone = value.tzinfo found: dict[str, object] = {"fold": value.fold} if value.fold else {} - if zone is None: - return found + if zone is None or value.utcoffset() is None: + raise _refused("a naive datetime cannot cross a contract: its time zone is explicit") if type(zone) is ZoneInfo and zone.key is not None: return {**found, "zone": zone.key} - if type(zone) is timezone: + if type(zone) in OFFSET_KINDS: offset = zone.utcoffset(None) - return {**found, "offset": [offset.days, offset.seconds, offset.microseconds]} + if offset is None or (type(zone) is TzInfo and offset.microseconds): + raise _refused(f"a {qualified(type(zone))} without a whole offset cannot cross") + return { + **found, + "offset": [offset.days, offset.seconds, offset.microseconds], + "kind": OFFSET_KINDS[type(zone)], + } raise _refused(f"a {qualified(type(zone))} zone cannot cross a contract") -def _tzinfo(tree: dict[str, Any]) -> ZoneInfo | timezone | None: +def _tzinfo(tree: dict[str, Any]) -> tzinfo: if "zone" in tree: return ZoneInfo(tree["zone"]) - if "offset" in tree: - days, seconds, microseconds = tree["offset"] - return timezone(timedelta(days=days, seconds=seconds, microseconds=microseconds)) - return None + days, seconds, microseconds = tree["offset"] + offset = timedelta(days=days, seconds=seconds, microseconds=microseconds) + if tree["kind"] == "pydantic": + return TzInfo(int(offset.total_seconds())) + if tree["kind"] == "timezone": + return timezone(offset) + raise ValueError(f"unknown offset kind {tree['kind']!r}") diff --git a/packages/contracts/src/orvex/contracts/context.py b/packages/contracts/src/orvex/contracts/context.py new file mode 100644 index 00000000..9e9810f7 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/context.py @@ -0,0 +1,65 @@ +"""The call context (#486 §2 item 6): who calls, for which project and revision, under which +correlation id; built once at each entry point and carried, per call, into the envelope. + +A context holds real values only: a field that is absent, not a string, blank or a placeholder +(`envelope.PLACEHOLDERS`) is refused when the context is built (CONTRACT_CONTEXT_INVALID), and +the binding refuses the same in every envelope, so a placeholder never reaches an +implementation either way. `CallContext.envelope` is the one way a consumer makes an envelope. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import TYPE_CHECKING + +from .envelope import CallEnvelope, real_value +from .interface_errors import ContractCallRefused + +if TYPE_CHECKING: + from .interface import InterfaceContract + from .version import Version + + +@dataclass(frozen=True, slots=True) +class CallContext: + """The caller's side of every envelope: the same for each call of one request or command.""" + + correlation_id: str + actor: str + project_id: str + revision: str + + def __post_init__(self) -> None: + for field in fields(self): + if not real_value(getattr(self, field.name)): + raise ContractCallRefused( + "CONTRACT_CONTEXT_INVALID", + f"a call context needs a real {field.name}, not a missing or placeholder one", + {"field": field.name}, + ) + + def envelope( + self, + contract: InterfaceContract, + operation: str, + idempotency_key: str | None = None, + version: Version | None = None, + ) -> CallEnvelope: + """The envelope of one call of `operation`, at `version` (the contract's by default); + an operation that changes state needs its idempotency key.""" + declared = contract.operation(operation) + if declared.changes_state and not real_value(idempotency_key): + raise ContractCallRefused( + "CONTRACT_ENVELOPE_INVALID", + f"{contract.name}.{operation} changes state and needs an idempotency key", + {"contract": contract.name, "operation": operation, "field": "idempotency_key"}, + ) + return CallEnvelope( + contract_name=contract.name, + contract_version=str(contract.version if version is None else version), + correlation_id=self.correlation_id, + actor=self.actor, + project_id=self.project_id, + revision=self.revision, + idempotency_key=idempotency_key, + ) diff --git a/packages/contracts/src/orvex/contracts/envelope.py b/packages/contracts/src/orvex/contracts/envelope.py index 3a0b8420..07310a99 100644 --- a/packages/contracts/src/orvex/contracts/envelope.py +++ b/packages/contracts/src/orvex/contracts/envelope.py @@ -10,11 +10,13 @@ - `project_id` and `revision`, the project and the revision the call acts on; - `idempotency_key`, required on every operation that changes state. -A field that is absent, not a string or blank, a key missing on an operation that changes -state, or an envelope for another contract is refused with CONTRACT_ENVELOPE_INVALID. A version -the provider cannot serve is refused with CONTRACT_VERSION_UNSUPPORTED: another major version, -or an operation the provider's version or the caller's version does not have. A caller at a -newer minor version is served for the operations the provider has (N against N-1). +A field that is absent, not a string, blank or a placeholder (`PLACEHOLDERS`), a key missing +on an operation that changes state, or an envelope for another contract is refused with +CONTRACT_ENVELOPE_INVALID. A version the provider cannot serve is refused with +CONTRACT_VERSION_UNSUPPORTED: another major version, or an operation the provider's version or +the caller's version does not have. A caller at a newer minor version is served for the +operations the provider has (N against N-1). Consumers make envelopes from a `CallContext` +(`orvex.contracts.context`). """ from __future__ import annotations @@ -52,8 +54,33 @@ class CallEnvelope: idempotency_key: str | None = None -def _present(value: object) -> bool: - return type(value) is str and bool(value.strip()) +# Values that stand in for a missing one: never accepted as a field of a context or envelope. +PLACEHOLDERS = frozenset( + { + "-", + "?", + "anonymous", + "changeme", + "default", + "dummy", + "n/a", + "na", + "nil", + "none", + "null", + "placeholder", + "tbd", + "todo", + "undefined", + "unknown", + "xxx", + } +) + + +def real_value(value: object) -> bool: + """A string that is neither blank nor a placeholder (compared without case or spaces).""" + return type(value) is str and bool(value.strip()) and value.strip().lower() not in PLACEHOLDERS def envelope_refusal( @@ -72,9 +99,9 @@ def invalid(field: str, text: str) -> ContractCallRefused: if type(envelope) is not CallEnvelope: return invalid("envelope", "a call carries a CallEnvelope as its first argument") for field in REQUIRED: - if not _present(getattr(envelope, field, None)): + if not real_value(getattr(envelope, field, None)): return invalid(field, f"the envelope has no {field}") - if operation.changes_state and not _present(envelope.idempotency_key): + if operation.changes_state and not real_value(envelope.idempotency_key): return invalid(IDEMPOTENCY_KEY, "an operation that changes state needs an idempotency key") if envelope.idempotency_key is not None and type(envelope.idempotency_key) is not str: return invalid(IDEMPOTENCY_KEY, "an idempotency key is a string") diff --git a/packages/contracts/src/orvex/contracts/interface_errors.py b/packages/contracts/src/orvex/contracts/interface_errors.py index 2a8257b8..5d7e4875 100644 --- a/packages/contracts/src/orvex/contracts/interface_errors.py +++ b/packages/contracts/src/orvex/contracts/interface_errors.py @@ -1,7 +1,8 @@ """The errors of the interface-contract machinery: a stable code, a message and details. `ContractCallRefused` is the typed refusal of one call before the implementation runs: -CONTRACT_ENVELOPE_INVALID and CONTRACT_VERSION_UNSUPPORTED (`orvex.contracts.envelope`). +CONTRACT_ENVELOPE_INVALID and CONTRACT_VERSION_UNSUPPORTED (`orvex.contracts.envelope`), and +CONTRACT_CONTEXT_INVALID for a call context that is not real (`orvex.contracts.context`). Every other code is an `InterfaceContractError`: INTERFACE_CONTRACT_INVALID, INTERFACE_VERSION_INVALID, INTERFACE_TYPE_NOT_SERIALIZABLE, INTERFACE_UNSUPPORTED_OPERATION_DECLARED, INTERFACE_OPERATION_UNKNOWN, diff --git a/packages/contracts/src/orvex/contracts/kit_client.py b/packages/contracts/src/orvex/contracts/kit_client.py index e2cf1456..6122bb11 100644 --- a/packages/contracts/src/orvex/contracts/kit_client.py +++ b/packages/contracts/src/orvex/contracts/kit_client.py @@ -1,5 +1,5 @@ """What a kit case calls an implementation through (#486 G1, G3): its binding, with a complete -envelope made for each call at the run's consumer version. +envelope made from the case's own call context for each call, at the run's consumer version. `client.(...)` sends a new envelope, with a fresh idempotency key when the operation changes state; `client.envelope(operation)` and `client.send(operation, envelope, ...)` let a @@ -14,6 +14,7 @@ from typing import Any from .binding import Bound +from .context import CallContext from .envelope import CallEnvelope from .version import Version @@ -21,12 +22,18 @@ class KitClient: """One case's caller: a consumer at `consumer` reaching the implementation through `bound`.""" - __slots__ = ("_bound", "_case", "_consumer", "_keys") + __slots__ = ("_bound", "_case", "_consumer", "_context", "_keys") def __init__(self, bound: Bound, consumer: Version, case: str) -> None: self._bound = bound self._consumer = consumer self._case = case + self._context = CallContext( + correlation_id=f"kit-{case}", + actor="kit-actor", + project_id="kit-project", + revision="kit-revision", + ) self._keys = 0 @property @@ -34,21 +41,13 @@ def consumer(self) -> Version: return self._consumer def envelope(self, operation: str, **changes: Any) -> CallEnvelope: # noqa: ANN401 - """A complete envelope for one call of `operation`, with `changes` applied (a case may - plant any value, a missing field included).""" + """A complete envelope for one call of `operation`, made from the case's context, with + `changes` applied (a case may plant any value, a missing field included).""" key = None if self._bound.contract.operation(operation).changes_state: self._keys += 1 key = f"kit-{self._case}-{self._keys}" - envelope = CallEnvelope( - contract_name=self._bound.contract.name, - contract_version=str(self._consumer), - correlation_id=f"kit-{self._case}", - actor="kit-actor", - project_id="kit-project", - revision="kit-revision", - idempotency_key=key, - ) + envelope = self._context.envelope(self._bound.contract, operation, key, self._consumer) return dataclasses.replace(envelope, **changes) def send( diff --git a/packages/contracts/src/orvex/contracts/kit_run.py b/packages/contracts/src/orvex/contracts/kit_run.py index a7cac003..cda34037 100644 --- a/packages/contracts/src/orvex/contracts/kit_run.py +++ b/packages/contracts/src/orvex/contracts/kit_run.py @@ -5,10 +5,10 @@ version `consumer`, which runs the cases that exist at its version. A case that exists at the provided version must pass. A newer case (a consumer at N against a provider at N-1) must pass or be refused with CONTRACT_VERSION_UNSUPPORTED: never a wrong answer. Each run also sends the -envelope cases the contract implies, which no implementation can pass by itself: a call missing -each envelope field, or without its idempotency key where it changes state, is refused with -CONTRACT_ENVELOPE_INVALID, and a call at another major version with -CONTRACT_VERSION_UNSUPPORTED, both before the implementation is reached. +envelope cases the contract implies, which no implementation can pass by itself: a call whose +envelope field is absent, blank or a placeholder, or that lacks its idempotency key where it +changes state, is refused with CONTRACT_ENVELOPE_INVALID, and a call at another major version +with CONTRACT_VERSION_UNSUPPORTED, both before the implementation is reached. """ from __future__ import annotations @@ -112,6 +112,7 @@ def _envelope_changes( for field in REQUIRED: yield f"{field}-missing", "CONTRACT_ENVELOPE_INVALID", {field: None} yield f"{field}-blank", "CONTRACT_ENVELOPE_INVALID", {field: " "} + yield f"{field}-placeholder", "CONTRACT_ENVELOPE_INVALID", {field: "unknown"} if operation.changes_state: yield f"{IDEMPOTENCY_KEY}-missing", "CONTRACT_ENVELOPE_INVALID", {IDEMPOTENCY_KEY: None} yield "other-contract", "CONTRACT_ENVELOPE_INVALID", {"contract_name": f"{contract.name}-x"} diff --git a/packages/contracts/src/orvex/contracts/module.toml b/packages/contracts/src/orvex/contracts/module.toml index 473e97ab..0f16d599 100644 --- a/packages/contracts/src/orvex/contracts/module.toml +++ b/packages/contracts/src/orvex/contracts/module.toml @@ -1,7 +1,8 @@ # The manifest of this module (#486 §2.5, orvex.contracts.manifest): it implements no interface -# contract and requires none; the contract types depend on pydantic, Arrow tables on pyarrow. +# contract and requires none; the contract types depend on pydantic, Arrow tables on pyarrow, and +# aware datetimes parsed by pydantic on its `TzInfo` (pydantic-core). module = "contracts" [dependencies] -runtime = ["pyarrow", "pydantic"] +runtime = ["pyarrow", "pydantic", "pydantic-core"] tests = ["pytest"] diff --git a/packages/contracts/src/orvex/contracts/serializable.py b/packages/contracts/src/orvex/contracts/serializable.py index e7e0ef3f..1a653f46 100644 --- a/packages/contracts/src/orvex/contracts/serializable.py +++ b/packages/contracts/src/orvex/contracts/serializable.py @@ -1,11 +1,12 @@ """Which annotated types may cross an interface contract (#486 G5): plain values, never live objects, so every operation can later run in another process (#156). -A value type is one of `SCALARS` (an Arrow table among them), a `Literal` of booleans, -integers, strings or `None`, a union of value types, a parameterized `tuple`, `list` or `dict` -with `str` keys, an `Enum`, or a pydantic model (the generated contract types) or frozen -dataclass whose fields are all value types. Anything else (`Any`, `object`, callables, -iterators, locks, connections, a mutable dataclass, a bare `tuple`) is refused. +A value type is one of `SCALARS` (an aware datetime and an Arrow table among them), a +`Literal` of booleans, integers, strings or `None`, a union of value types, a value type with +constraints (`Annotated`), a parameterized `tuple`, `list` or `dict` with `str` keys, an `Enum`, +or a pydantic model (the generated contract types) or frozen dataclass whose fields are all +value types. Anything else (`Any`, `object`, callables, iterators, locks, connections, a +mutable dataclass, a bare `tuple`) is refused. """ from __future__ import annotations @@ -18,7 +19,7 @@ from enum import Enum import pyarrow as pa -from pydantic import BaseModel +from pydantic import AwareDatetime, BaseModel SCALARS: tuple[type, ...] = ( type(None), @@ -29,6 +30,7 @@ bytes, Decimal, datetime, + AwareDatetime, date, time, timedelta, @@ -67,6 +69,8 @@ def crossing_problem(annotation: object, classes: set[type]) -> str | None: return None origin = typing.get_origin(annotation) arguments = typing.get_args(annotation) + if origin is typing.Annotated: # constraints narrow a value type; they never widen it + return crossing_problem(arguments[0], classes) if origin in (typing.Union, types.UnionType): return _first(crossing_problem(argument, classes) for argument in arguments) if origin is typing.Literal: @@ -79,7 +83,10 @@ def crossing_problem(annotation: object, classes: set[type]) -> str | None: return f"{type_name(annotation)} does not say what it holds" return _first(crossing_problem(item, classes) for item in items) if origin is dict: - if not arguments or arguments[0] is not str: + key = arguments[0] if arguments else None + if typing.get_origin(key) is typing.Annotated: + key = typing.get_args(key)[0] + if key is not str: return f"{type_name(annotation)}: only `str` keys may cross" return crossing_problem(arguments[1], classes) if origin is None and isinstance(annotation, type): diff --git a/packages/contracts/tests/test_binding.py b/packages/contracts/tests/test_binding.py index c93a70cd..2f18fdc8 100644 --- a/packages/contracts/tests/test_binding.py +++ b/packages/contracts/tests/test_binding.py @@ -16,7 +16,9 @@ from orvex.contracts.envelope import CallEnvelope from orvex.contracts.interface_errors import ContractCallRefused, InterfaceContractError from orvex.contracts.kit_run import run_kit, verify_kit -from pydantic import BaseModel, Field +from orvex.contracts.store.v1 import Snapshot +from pydantic import AwareDatetime, BaseModel, Field +from pydantic_core import TzInfo from .sample_contract import ( ENVELOPE, @@ -37,6 +39,15 @@ class Shape(BaseModel): side: Decimal = Field(gt=0) +class Stamp(BaseModel): + """A generated-style model whose datetime must be aware.""" + + at: AwareDatetime + + +OFFSET = TzInfo(-4 * 3600) + + class Moment(BaseModel): """A model holding an IANA zone: its own JSON would turn the zone into a fixed offset.""" @@ -69,7 +80,7 @@ class ReadingChild(Reading): pass -CODEC = Codec(frozenset({Reading, Color, Shape, Moment, CallEnvelope})) +CODEC = Codec(frozenset({Reading, Color, Shape, Moment, Stamp, Snapshot, CallEnvelope})) VALUES: list[object] = [ None, @@ -89,7 +100,7 @@ class ReadingChild(Reading): Decimal("123456789012345678901234567890.123456789"), datetime(2026, 3, 8, 1, 30, tzinfo=ZoneInfo("America/La_Paz")), datetime(2026, 11, 1, 1, 30, fold=1, tzinfo=ZoneInfo("America/New_York")), - datetime(2026, 3, 8, 1, 30, 0, 123456), + datetime(2026, 3, 8, 1, 30, 0, 123456, tzinfo=OFFSET), datetime(2026, 3, 8, tzinfo=UTC), datetime(2026, 3, 8, tzinfo=timezone(timedelta(hours=-4))), date(2026, 2, 28), @@ -106,6 +117,10 @@ class ReadingChild(Reading): color=Color.RED, readings=(READING,), ), + Stamp(at=datetime(2026, 3, 8, 1, 30, tzinfo=ZoneInfo("America/La_Paz"))), + Stamp(at=datetime(2026, 3, 8, 1, 30, tzinfo=timezone(timedelta(hours=5, minutes=45)))), + Stamp.model_validate({"at": "2026-03-08T01:30:00-04:00"}), + Snapshot.model_validate({"kind": "snapshot", "id": 7, "committed_at": "2026-03-08T01:30:00Z"}), TABLE, TABLE.slice(0, 0), READING, @@ -137,6 +152,7 @@ def same(left: object, right: object) -> bool: assert isinstance(right, (datetime, time)) return ( left == right + and type(left.tzinfo) is type(right.tzinfo) and str(left.tzinfo) == str(right.tzinfo) and left.fold == right.fold and left.isoformat() == right.isoformat() @@ -170,6 +186,14 @@ def test_an_arrow_table_keeps_its_types_nulls_and_zones(self) -> None: assert back.column("huge").to_pylist()[0] == Decimal(2**126) assert back.schema.field("at").type == pa.timestamp("us", "America/La_Paz") + def test_a_parsed_fixed_offset_comes_back_as_pydantic_s_own_zone(self) -> None: + stamp = Stamp.model_validate({"at": "2026-03-08T01:30:00-04:00"}) + assert type(stamp.at.tzinfo) is TzInfo + back = CODEC.decode(CODEC.encode(stamp)) + assert isinstance(back, Stamp) + assert type(back.at.tzinfo) is TzInfo + assert back.at.utcoffset() == timedelta(hours=-4) + def test_a_zone_comes_back_as_the_same_zone(self) -> None: back = CODEC.decode(CODEC.encode(READING.at)) assert isinstance(back, datetime) @@ -189,6 +213,9 @@ def test_a_zone_comes_back_as_the_same_zone(self) -> None: {1, 2}, bytearray(b"x"), time(1, tzinfo=ZoneInfo("Europe/Madrid")), + datetime(2026, 3, 8, 1, 30), + (datetime(2026, 3, 8),), + Moment.model_construct(at=datetime(2026, 3, 8), color=Color.RED, readings=()), ], ids=lambda value: type(value).__name__, ) diff --git a/packages/contracts/tests/test_context.py b/packages/contracts/tests/test_context.py new file mode 100644 index 00000000..5d2607dc --- /dev/null +++ b/packages/contracts/tests/test_context.py @@ -0,0 +1,63 @@ +"""The call context (#486 §2 item 6): real values only, and the one way a consumer makes an +envelope.""" + +from __future__ import annotations + +import dataclasses +from typing import Any + +import pytest +from orvex.contracts.binding import Bound +from orvex.contracts.context import CallContext +from orvex.contracts.envelope import PLACEHOLDERS +from orvex.contracts.interface_errors import ContractCallRefused +from orvex.contracts.version import Version + +from .sample_contract import LEDGER, READING, MemoryLedger + +CONTEXT = CallContext( + correlation_id="4f1c2a9e", + actor="user:ana", + project_id="billing", + revision="a1b2c3d", +) +FIELDS = [field.name for field in dataclasses.fields(CallContext)] + + +@pytest.mark.parametrize("field", FIELDS) +@pytest.mark.parametrize("value", [None, "", " ", 1, *sorted(PLACEHOLDERS), " Unknown "]) +def test_a_missing_or_placeholder_value_is_refused(field: str, value: object) -> None: + planted: dict[str, Any] = {field: value} + with pytest.raises(ContractCallRefused) as raised: + dataclasses.replace(CONTEXT, **planted) + assert raised.value.code == "CONTRACT_CONTEXT_INVALID" + assert raised.value.details == {"field": field} + + +def test_the_envelope_carries_the_context_and_the_contract_s_version() -> None: + envelope = CONTEXT.envelope(LEDGER, "names") + assert envelope.contract_name == "ledger" + assert envelope.contract_version == "1.1" + assert (envelope.correlation_id, envelope.actor, envelope.project_id, envelope.revision) == ( + "4f1c2a9e", + "user:ana", + "billing", + "a1b2c3d", + ) + assert envelope.idempotency_key is None + assert CONTEXT.envelope(LEDGER, "names", version=Version(1, 0)).contract_version == "1.0" + + +@pytest.mark.parametrize("key", [None, "", "unknown"]) +def test_an_operation_that_changes_state_needs_a_real_key(key: str | None) -> None: + with pytest.raises(ContractCallRefused) as raised: + CONTEXT.envelope(LEDGER, "record", key) + assert raised.value.code == "CONTRACT_ENVELOPE_INVALID" + assert raised.value.details["field"] == "idempotency_key" + + +@pytest.mark.parametrize("serialized", [False, True]) +def test_an_envelope_from_the_context_is_served(serialized: bool) -> None: + bound = Bound(LEDGER, MemoryLedger(), serialized=serialized) + assert bound.record(CONTEXT.envelope(LEDGER, "record", "order-17"), READING) == 1 + assert bound.names(CONTEXT.envelope(LEDGER, "names")) == (READING.name,) diff --git a/packages/contracts/tests/test_envelope.py b/packages/contracts/tests/test_envelope.py index 945400d4..23183043 100644 --- a/packages/contracts/tests/test_envelope.py +++ b/packages/contracts/tests/test_envelope.py @@ -40,7 +40,11 @@ def refused( @BINDINGS @pytest.mark.parametrize("field", REQUIRED) -@pytest.mark.parametrize("value", [None, "", " ", 7], ids=["absent", "empty", "blank", "int"]) +@pytest.mark.parametrize( + "value", + [None, "", " ", 7, "unknown", " N/A ", "placeholder"], + ids=["absent", "empty", "blank", "int", "unknown", "n/a", "placeholder"], +) def test_a_call_missing_a_required_field_is_refused( serialized: bool, field: str, value: object ) -> None: diff --git a/packages/contracts/tests/test_interface.py b/packages/contracts/tests/test_interface.py index ac58ab4d..d08c298a 100644 --- a/packages/contracts/tests/test_interface.py +++ b/packages/contracts/tests/test_interface.py @@ -16,8 +16,10 @@ from orvex.contracts.envelope import CallEnvelope from orvex.contracts.interface import InterfaceContract, Operation from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.serializable import crossing_problem +from orvex.contracts.store import v1 as store_v1 from orvex.contracts.version import Version -from pydantic import BaseModel +from pydantic import AwareDatetime, BaseModel from .sample_contract import LEDGER, LedgerError, Reading @@ -237,6 +239,7 @@ def test_an_operation_without_the_envelope_first_is_refused( SERIALIZABLE: list[object] = [ + AwareDatetime, int, bool, float, @@ -350,3 +353,17 @@ def test_the_format_expresses_the_result_cache_that_g1_substitutes() -> None: assert cache.describe()["operations"]["get"]["returns"] == ( # type: ignore[index] f"{Stored.__module__}.Stored | None" ) + + +def test_every_generated_store_type_may_cross() -> None: + models = [ + kind + for kind in vars(store_v1).values() + if isinstance(kind, type) + and issubclass(kind, BaseModel) + and kind.__module__ == store_v1.__name__ + ] + assert models + assert {kind.__name__: crossing_problem(kind, set()) for kind in models} == { + kind.__name__: None for kind in models + } diff --git a/packages/contracts/tests/test_kit.py b/packages/contracts/tests/test_kit.py index 68159cad..fc24d884 100644 --- a/packages/contracts/tests/test_kit.py +++ b/packages/contracts/tests/test_kit.py @@ -177,7 +177,7 @@ def test_every_operation_gets_its_envelope_cases(self) -> None: assert "envelope.names.idempotency_key-missing" not in ids assert "envelope.count.correlation_id-blank" in ids assert "envelope.readings.major-2" in ids - assert len(ids) == 4 * 14 + 1 + assert len(ids) == 4 * 20 + 1 older = run_kit(LEDGER_KIT, MemoryLedger, serialized=False, consumer=V10) assert not any(r.id.startswith("envelope.count.") for r in older.envelope_results) diff --git a/packages/contracts/tests/test_manifest.py b/packages/contracts/tests/test_manifest.py index 7a6e3ac0..d52d46f5 100644 --- a/packages/contracts/tests/test_manifest.py +++ b/packages/contracts/tests/test_manifest.py @@ -45,7 +45,7 @@ def test_the_shipped_manifest_of_the_contracts_distribution_is_valid() -> None: assert manifest.module == "contracts" assert manifest.provides == () assert manifest.requires == () - assert manifest.runtime == ("pyarrow", "pydantic") + assert manifest.runtime == ("pyarrow", "pydantic", "pydantic-core") def test_provides_and_requires_may_be_absent() -> None: diff --git a/packages/server/src/orvex/server/app.py b/packages/server/src/orvex/server/app.py index 6388ecf8..78dbaeed 100644 --- a/packages/server/src/orvex/server/app.py +++ b/packages/server/src/orvex/server/app.py @@ -5,7 +5,6 @@ import json import logging import time -import uuid from collections.abc import Awaitable, Callable from dataclasses import dataclass from pathlib import Path @@ -21,6 +20,7 @@ from orvex.contracts.http_api import v1 from orvex.semantic.errors import SemanticError from orvex.server import lake_refusals, semantic_delivery, semantic_http, semantic_jobs +from orvex.server.call_context import REQUEST_ID_HEADER, request_correlation_id from orvex.server.local_access import CODE_PARAMETER, LocalAccess from orvex.server.route_permissions import checked_permissions from orvex.server.versions import Versions @@ -28,7 +28,6 @@ from starlette.concurrency import run_in_threadpool from starlette.exceptions import HTTPException -REQUEST_ID_HEADER = "X-Request-Id" API_PREFIX = "/api/" # Local mode answers only requests addressed to itself by one of these names (DNS rebinding). LOCAL_NAMES = ("127.0.0.1", "localhost") @@ -301,7 +300,7 @@ def web_page(path: str) -> Path | None: async def request_context( request: Request, call_next: Callable[[Request], Awaitable[Response]] ) -> Response: - request.state.request_id = uuid.uuid4().hex + request.state.request_id = request_correlation_id(request.headers) started = time.perf_counter() try: # Before any route, the web build or the 404 handling. A semantic request is diff --git a/packages/server/src/orvex/server/call_context.py b/packages/server/src/orvex/server/call_context.py new file mode 100644 index 00000000..1b35b8a7 --- /dev/null +++ b/packages/server/src/orvex/server/call_context.py @@ -0,0 +1,42 @@ +"""The call context of each server entry point (#486 §2 item 6): one correlation id per HTTP +request, CLI command or scheduled run, and the `CallContext` every contract call of it carries. + +An HTTP request keeps the caller's `X-Request-Id` when it is a well-formed id (8 to 128 letters, +digits, `.`, `_` or `-`) and gets a new one otherwise, so the id that answers, logs and calls +carry is one and the same. A command or a scheduled run gets a new id. The actor, project and +revision come from the entry point, which resolves them; `CallContext` refuses a missing or +placeholder value (CONTRACT_CONTEXT_INVALID). +""" + +from __future__ import annotations + +import re +import uuid +from collections.abc import Mapping + +from fastapi import Request +from orvex.contracts.context import CallContext + +REQUEST_ID_HEADER = "X-Request-Id" +REQUEST_ID = re.compile(r"[A-Za-z0-9._-]{8,128}") + + +def new_correlation_id() -> str: + return uuid.uuid4().hex + + +def request_correlation_id(headers: Mapping[str, str]) -> str: + """The caller's request id when it is well formed, a new one otherwise.""" + given = headers.get(REQUEST_ID_HEADER) + return given if given is not None and REQUEST_ID.fullmatch(given) else new_correlation_id() + + +def request_context(request: Request, actor: str, project_id: str, revision: str) -> CallContext: + """The context of the contract calls one HTTP request makes, under its request id.""" + correlation: str = request.state.request_id + return CallContext(correlation, actor, project_id, revision) + + +def command_context(actor: str, project_id: str, revision: str) -> CallContext: + """The context of the contract calls one CLI command or scheduled run makes.""" + return CallContext(new_correlation_id(), actor, project_id, revision) diff --git a/packages/server/src/orvex/server/server_app.py b/packages/server/src/orvex/server/server_app.py index eaf1e493..a1c81808 100644 --- a/packages/server/src/orvex/server/server_app.py +++ b/packages/server/src/orvex/server/server_app.py @@ -28,7 +28,6 @@ import logging import time -import uuid from collections.abc import Awaitable, Callable from dataclasses import dataclass @@ -53,12 +52,12 @@ from orvex.server import oidc_http, semantic_http, sign_in_http from orvex.server.admin_http import Administration, mount_administration from orvex.server.app import ( - REQUEST_ID_HEADER, error_response, install_error_handlers, mount_semantic, mount_status, ) +from orvex.server.call_context import REQUEST_ID_HEADER, request_correlation_id from orvex.server.route_permissions import ( ADMINISTRATION, allows, @@ -104,7 +103,7 @@ def create_server_app( async def request_rules( request: Request, call_next: Callable[[Request], Awaitable[Response]] ) -> Response: - request.state.request_id = uuid.uuid4().hex + request.state.request_id = request_correlation_id(request.headers) started = time.perf_counter() try: refused = await rules(request) diff --git a/packages/server/tests/test_call_context.py b/packages/server/tests/test_call_context.py new file mode 100644 index 00000000..4945231e --- /dev/null +++ b/packages/server/tests/test_call_context.py @@ -0,0 +1,79 @@ +"""The call context of each server entry point (#486 §2 item 6): one real correlation id per +request or command, and a context that refuses missing or placeholder values.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from orvex.contracts.interface_errors import ContractCallRefused +from orvex.server.app import create_app +from orvex.server.call_context import ( + command_context, + request_context, + request_correlation_id, +) +from orvex.server.versions import Versions +from starlette.requests import Request + +VERSIONS = Versions("0.1.0", (("server", "0.1.0"),), (("http-api", "v1"),)) +PORT = 7878 + + +@pytest.fixture +def client(tmp_path: Path) -> TestClient: + (tmp_path / "index.html").write_text("app") + app = create_app(VERSIONS, tmp_path, PORT) + return TestClient(app, base_url=f"http://127.0.0.1:{PORT}") + + +def test_a_well_formed_request_id_is_kept_end_to_end(client: TestClient) -> None: + response = client.get("/api/v1/health", headers={"X-Request-Id": "trace-0042.a_b"}) + assert response.headers["X-Request-Id"] == "trace-0042.a_b" + + +@pytest.mark.parametrize( + "given", [None, "", "short", "x" * 129, "bad id with spaces", "id\r\nX-Evil: 1", "ñandú-1234"] +) +def test_a_missing_or_malformed_request_id_is_replaced(given: str | None) -> None: + headers = {} if given is None else {"X-Request-Id": given} + found = request_correlation_id(headers) + assert found != given + assert len(found) == 32 + assert found != request_correlation_id(headers) + + +def test_each_request_gets_its_own_id(client: TestClient) -> None: + first = client.get("/api/v1/health").headers["X-Request-Id"] + second = client.get("/api/v1/health").headers["X-Request-Id"] + assert first != second + + +def test_a_request_s_context_carries_its_request_id() -> None: + request = Request({"type": "http", "headers": [], "state": {"request_id": "trace-0042"}}) + context = request_context(request, "user:ana", "billing", "a1b2c3d") + assert context.correlation_id == "trace-0042" + assert (context.actor, context.project_id, context.revision) == ( + "user:ana", + "billing", + "a1b2c3d", + ) + + +def test_each_command_gets_its_own_correlation_id() -> None: + first = command_context("scheduler:instance-1", "billing", "a1b2c3d") + second = command_context("scheduler:instance-1", "billing", "a1b2c3d") + assert first.correlation_id != second.correlation_id + + +@pytest.mark.parametrize( + ("actor", "project", "revision"), + [("", "billing", "r1"), ("unknown", "billing", "r1"), ("user:ana", "n/a", "r1")], +) +def test_an_entry_point_cannot_build_a_placeholder_context( + actor: str, project: str, revision: str +) -> None: + with pytest.raises(ContractCallRefused) as raised: + command_context(actor, project, revision) + assert raised.value.code == "CONTRACT_CONTEXT_INVALID" From 20f13306b81a3170046ebf43256ff918664adb50 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 08:22:00 -0400 Subject: [PATCH 10/21] build(repo): pin pydantic-core in the runtime group, now a declared dependency of orvex-contracts Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f4f7f9ac..21db1fed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ runtime = [ "pg8000==1.31.6", "pyarrow==26.0.1", "pydantic==2.14.3", + "pydantic-core==2.50.4", "pyyaml==6.0.3", "tzdata==2026.6", "uvicorn==0.54.0", From 6ba558b1507658dc8db90434a428d4f90d31622c Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 08:40:08 -0400 Subject: [PATCH 11/21] feat(contracts): a kit case may run in a process of its own, built from its declared environment A case declares variables and an extension directory (copied with or without given extensions); the kit runs it in a child process built from that declaration alone and reads its result back through the serializing codec, so no kit patches an implementation's internals and production holds no switch for a test. One case's run moves to kit_case.py, shared by the kit and the child. Plant: a case whose declared environment is ignored fails. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- ...-interface-contracts-kits-and-manifests.md | 2 +- docs/process/git-workflow.md | 2 + packages/contracts/README.md | 2 +- packages/contracts/src/orvex/contracts/kit.py | 52 ++++- .../contracts/src/orvex/contracts/kit_case.py | 91 +++++++++ .../src/orvex/contracts/kit_process.py | 180 ++++++++++++++++++ .../contracts/src/orvex/contracts/kit_run.py | 75 +------- .../contracts/tests/environment_contract.py | 106 +++++++++++ packages/contracts/tests/test_kit.py | 1 + packages/contracts/tests/test_kit_process.py | 107 +++++++++++ 10 files changed, 550 insertions(+), 68 deletions(-) create mode 100644 packages/contracts/src/orvex/contracts/kit_case.py create mode 100644 packages/contracts/src/orvex/contracts/kit_process.py create mode 100644 packages/contracts/tests/environment_contract.py create mode 100644 packages/contracts/tests/test_kit_process.py diff --git a/docs/adr/0033-interface-contracts-kits-and-manifests.md b/docs/adr/0033-interface-contracts-kits-and-manifests.md index 1c284699..6dce8d62 100644 --- a/docs/adr/0033-interface-contracts-kits-and-manifests.md +++ b/docs/adr/0033-interface-contracts-kits-and-manifests.md @@ -8,7 +8,7 @@ ADR-0004 makes modules talk only through contracts, but its contracts are data c ## Decision 1. **Interface contracts** (amends rule 2). Besides the language-neutral data contracts of `contracts/`, each module's operations are declared as an interface contract in `orvex.contracts`: a `typing.Protocol` whose public methods are the operations, and an `InterfaceContract` (`orvex.contracts.interface`) that names the contract, gives it a `MAJOR.MINOR` version (`orvex.contracts.version`), groups the operations into capabilities, lists the error codes each may raise through the contract's error type, marks the operations that change state and the version each exists from. Only values cross: the annotated inputs and outputs are plain values, Arrow tables, enums, generated contract models and frozen dataclasses of them (`orvex.contracts.serializable`); no live object, connection or callable. The serializing codec keeps each exactly: a model field by field (its own JSON would turn an IANA zone into an offset), an Arrow table as Arrow IPC. Datetimes are aware: pydantic's `AwareDatetime` is a value type, an IANA zone and a fixed offset (`datetime.timezone`, or the `pydantic_core.TzInfo` pydantic parses an offset into) each come back as themselves, and a naive datetime is refused. `UNSUPPORTED_OPERATION` can never be declared: a capability is provided in full or not at all. The interface contract of module `` lives in `orvex.contracts..interface`, its kit in `orvex.contracts..kit` and its public test adapter, when it has one, in `orvex.contracts..adapter` (`orvex.contracts.registry`). 2. **The call envelope.** Every operation takes a `CallEnvelope` (`orvex.contracts.envelope`) as its first argument: `contract_name` and `contract_version` (the contract's version, never the module's or a physical format's), `correlation_id`, the effective `actor` or a verifiable reference to it, `project_id` and `revision`, and an `idempotency_key`, required on every operation that changes state. Every binding (`orvex.contracts.binding`) validates it before the implementation runs and fails closed with a typed `ContractCallRefused`: CONTRACT_ENVELOPE_INVALID for a missing, blank or mistyped field, a missing key or another contract's envelope; CONTRACT_VERSION_UNSUPPORTED for another major version or an operation that the provider's or the caller's version does not have. A caller at a newer minor version is served for the operations the provider has. A placeholder (`unknown`, `n/a`, `none` and the like, `envelope.PLACEHOLDERS`) is refused like a missing value. A consumer never writes an envelope by hand: it holds a `CallContext` (`orvex.contracts.context`: `correlation_id`, actor, `project_id`, revision, refused with CONTRACT_CONTEXT_INVALID when any is missing or a placeholder), built once at each server entry point (`orvex.server.call_context`: an HTTP request keeps a well-formed `X-Request-Id` as its correlation id or gets a new one, and a CLI command or scheduled run gets a new one), and every consumer client takes that context per call and makes the envelope from it (`CallContext.envelope`). The entry points build it where they first make a contract call, as each consumer moves onto its contract (#486 steps 2-7). -3. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise, their expectation in words and the version they exist from, a recorded case count and no skip list. A kit leaves no operation without a case, and each operation that changes state has an idempotency case: the call is sent twice with one key and its effect must happen exactly once. The runner (`orvex.contracts.kit_run`, `kit_client`, `codec`) runs every case on a fresh implementation, once in process and once through the serializing binding, by a consumer at N-1 and at N, and adds the envelope cases the contract implies (each missing field and key, another contract, another major version), which must be refused before the implementation is reached. A provider at N-1 runs the kit of N: the cases of its version pass, the newer ones pass or are refused with CONTRACT_VERSION_UNSUPPORTED, never answered wrongly. An implementation that passes its contract's whole kit may replace the module. +3. **Conformance kits.** Every interface contract ships a kit (`orvex.contracts.kit`): cases with a stable id, the operation they exercise, their expectation in words and the version they exist from, a recorded case count and no skip list. A kit leaves no operation without a case, and each operation that changes state has an idempotency case: the call is sent twice with one key and its effect must happen exactly once. The runner (`orvex.contracts.kit_run`, `kit_client`, `codec`) runs every case on a fresh implementation, once in process and once through the serializing binding, by a consumer at N-1 and at N, and adds the envelope cases the contract implies (each missing field and key, another contract, another major version), which must be refused before the implementation is reached. A provider at N-1 runs the kit of N: the cases of its version pass, the newer ones pass or are refused with CONTRACT_VERSION_UNSUPPORTED, never answered wrongly. A case may declare its own environment (variables, and an extension directory copied with or without given extensions): it then runs in a process of its own built from that declaration alone (`kit_process`), and its result returns through the serializing codec, so a kit reaches a condition such as a missing extension or an untested server version without patching an implementation's internals, and production never holds a switch for it. An implementation that passes its contract's whole kit may replace the module. 4. **Consumers know contracts, not modules** (amends rule 3). `server` stays the only composition root, but only `server/assembly.py` names implementations, and only through each module's public `api` package. Every other consumer (`server`, the conformance runners, `tools/`, the repository tests) imports only `orvex.contracts`. `tools/checks/consumers.py` enforces it, in report mode until #486 step 7 makes it blocking and removes the `server` exemption. 5. **Module manifests** (amends rule 4). Each module ships `module.toml` (`orvex.contracts.manifest`): the interface contracts it provides (version, capabilities, the `api` class that implements each) and requires (version, capabilities), and the distributions its code and its tests import. A module is named by its package path under the namespace, so an alternative implementation (`alternatives.result_cache_ipc`) has a manifest too. Before the server starts, `assembly.py` refuses (exit status 3, `ASSEMBLY_REFUSED`) any combination in which a required contract has no implementation, or several and the configuration chose none (or chose one that does not provide it); a version that does not provide the required one; a missing capability; an implementation that is not a class built from configuration by keyword arguments, lacks an operation of what it claims at its version, or whose method differs from the contract's signature, envelope included. Each module still owns its state alone (rule 4 unchanged); the manifest makes what it offers and needs explicit. `module.toml` does not replace `contracts.toml`: `contracts.toml` declares the language-neutral data contracts a module produces and consumes, by their `vN` versions (ADR-0004 rule 6, `versioning.md` §3), and the server checks them as before; `module.toml` declares the interface contracts, by `MAJOR.MINOR`, and the module's dependencies. Merging them would change the checked data-contract declarations of every module at once, which is outside #486. 6. **Compatibility of behavior.** The record of every interface contract and its kit (`packages/contracts/interface-record.json`, `tools/contracts/interface_record.py`) is compared with the record at the last product release (`tools/contracts/interface_compat.py`): within a major version operations, capabilities, error codes and kit cases with their expectations may only be added, each addition existing only from a version after the release; the signature of an existing operation (default values included) and the error codes it raises never change. A break needs a new major version (rule 6 of ADR-0004 applies to it). Rule 3 runs N against N-1 in both directions: each module's own tests run its kit by consumers at N-1 and N, and an implementation at N-1 runs the current kit with its own version as `provided`. The job that builds a module's last released implementation for that run arrives with the first released interface contract (#486 step 2); until then no released implementation exists. diff --git a/docs/process/git-workflow.md b/docs/process/git-workflow.md index cf9acc4e..7463f89c 100644 --- a/docs/process/git-workflow.md +++ b/docs/process/git-workflow.md @@ -127,7 +127,9 @@ packages/contracts/src/orvex/contracts/ generated contract types !packages/contracts/src/orvex/contracts/adapter.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/context.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/envelope.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/kit_case.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/kit_client.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/kit_process.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/version.py hand-written: counted (#486, interface contracts) tests/contracts/generated/ generated contract fixtures !tests/contracts/generated/__init__.py hand-written: counted diff --git a/packages/contracts/README.md b/packages/contracts/README.md index 2f6e07b7..c390b6b1 100644 --- a/packages/contracts/README.md +++ b/packages/contracts/README.md @@ -3,7 +3,7 @@ Every module may import this package. It holds: - Python types generated from [`contracts/`](../../contracts/README.md) by `python -m tools.contracts.generate` (`/v.py`), never edited by hand; -- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, state changes) and `version.py`; `envelope.py` (the call envelope every binding validates) and `context.py` (the call context envelopes are made from); `kit.py`, `kit_client.py` and `kit_run.py` (conformance kits: cases, recorded count, idempotency, no skips, run in process and serialized by consumers at N-1 and N); `binding.py`, `codec.py` and `serializable.py` (what may cross a contract); `adapter.py` (public test adapters); `manifest.py` (`module.toml`); `registry.py` and `interface_errors.py`; +- the interface-contract format (ADR-0033, #486): `interface.py` (operations, inputs, outputs, errors, capabilities, state changes) and `version.py`; `envelope.py` (the call envelope every binding validates) and `context.py` (the call context envelopes are made from); `kit.py`, `kit_client.py`, `kit_case.py`, `kit_process.py` and `kit_run.py` (conformance kits, cases with their own environment run in their own process: cases, recorded count, idempotency, no skips, run in process and serialized by consumers at N-1 and N); `binding.py`, `codec.py` and `serializable.py` (what may cross a contract); `adapter.py` (public test adapters); `manifest.py` (`module.toml`); `registry.py` and `interface_errors.py`; - `module.toml`, this module's manifest, and `interface-record.json` (beside `pyproject.toml`), the record of every interface contract that `python -m tools.contracts.interface_record` writes and `python -m tools.contracts.interface_compat` compares with the last release. Module pack: [`docs/modules/contracts.md`](../../docs/modules/contracts.md). diff --git a/packages/contracts/src/orvex/contracts/kit.py b/packages/contracts/src/orvex/contracts/kit.py index 0f40f190..717161f0 100644 --- a/packages/contracts/src/orvex/contracts/kit.py +++ b/packages/contracts/src/orvex/contracts/kit.py @@ -5,7 +5,10 @@ its expectation in words (the guarantee the compatibility check holds to, #486 G3), the version it exists from, and either a check that receives a `KitClient` and raises when the guarantee fails, or an `Idempotent` pair for an operation that changes state: the call is sent twice with -one idempotency key, and the effect must happen exactly once (#486 G3). The kit records how +one idempotency key, and the effect must happen exactly once (#486 G3). A case may declare its +own `CaseEnvironment` (variables and an extension directory): it then runs in a process of its +own built from that declaration (`kit_process`), so a kit never patches an implementation's +internals to reach a condition. The kit records how many cases it has; a kit whose cases differ from that count, that leaves an operation without a case, or an operation that changes state without an idempotency case, is refused when built. """ @@ -23,6 +26,7 @@ from .version import Version CASE_ID = re.compile(r"[a-z][a-z0-9]*([.-][a-z0-9]+)*") +VARIABLE = re.compile(r"[A-Z][A-Z0-9_]*") @dataclass(frozen=True, slots=True) @@ -34,6 +38,26 @@ class Idempotent: observe: Callable[[KitClient], object] +@dataclass(frozen=True, slots=True) +class ExtensionDirectory: + """A fresh directory named by the environment variable `variable`: a copy of the directory + the variable `source` names in the kit's own environment (empty when `source` is None), + without the files of the extensions in `without`.""" + + variable: str + source: str | None = None + without: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class CaseEnvironment: + """The environment a case runs in, in a process of its own built from this alone: its + variables, and at most one extension directory.""" + + variables: tuple[tuple[str, str], ...] = () + extensions: ExtensionDirectory | None = None + + @dataclass(frozen=True, slots=True) class KitCase: id: str @@ -41,6 +65,7 @@ class KitCase: expectation: str check: Callable[[KitClient], None] | Idempotent since: Version | None = None + environment: CaseEnvironment | None = None @dataclass(frozen=True, slots=True) @@ -102,12 +127,30 @@ def describe(self) -> dict[str, object]: "expectation": case.expectation, "idempotency": isinstance(case.check, Idempotent), "since": str(self.since(case)), + "environment": _environment(case.environment), } for case in self.cases } return {"recorded": self.recorded, "cases": cases} +def _environment(environment: CaseEnvironment | None) -> dict[str, object] | None: + """A case's environment as plain data: part of what its expectation means.""" + if environment is None: + return None + extensions = environment.extensions + return { + "variables": dict(environment.variables), + "extensions": None + if extensions is None + else { + "variable": extensions.variable, + "source": extensions.source, + "without": list(extensions.without), + }, + } + + def _case_problem(kit: Kit, case: KitCase) -> str | None: """What is wrong with a case of a declared operation: its version or its idempotency.""" contract = kit.contract @@ -117,4 +160,11 @@ def _case_problem(kit: Kit, case: KitCase) -> str | None: return f"since {since} is not between {contract.since(operation)} and {contract.version}" if isinstance(case.check, Idempotent) and not operation.changes_state: return "an idempotency case needs an operation that changes state" + environment = case.environment + if environment is not None: + names = [name for name, _ in environment.variables] + if environment.extensions is not None: + names.append(environment.extensions.variable) + if len(set(names)) != len(names) or not all(VARIABLE.fullmatch(n) for n in names): + return "its environment names each variable once, in UPPER_SNAKE_CASE" return None diff --git a/packages/contracts/src/orvex/contracts/kit_case.py b/packages/contracts/src/orvex/contracts/kit_case.py new file mode 100644 index 00000000..82caa886 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/kit_case.py @@ -0,0 +1,91 @@ +"""Running one kit case on a fresh implementation (#486 G1, G3): through one binding, by a +consumer at one version, against a provider at one version, with its outcome. + +A case passes, fails, or is refused: refused only when the binding itself raised +CONTRACT_VERSION_UNSUPPORTED at the top of the case, never when a refusal is found down a +`__cause__` chain or was raised by an implementation (which the binding turns into +INTERFACE_ERROR_UNDECLARED). A skip is a failure: the kit has no skips. An idempotency case +sends its call twice with one key and requires the effect exactly once. +""" + +from __future__ import annotations + +import functools +from collections.abc import Callable +from dataclasses import dataclass +from typing import Literal + +from .binding import Bound +from .interface_errors import ContractCallRefused +from .kit import Idempotent, Kit, KitCase +from .kit_client import KitClient +from .version import Version + +Outcome = Literal["passed", "refused", "failed"] +UNSUPPORTED = "CONTRACT_VERSION_UNSUPPORTED" + + +@dataclass(frozen=True, slots=True) +class CaseResult: + id: str + outcome: Outcome + detail: str + + @property + def passed(self) -> bool: + return self.outcome == "passed" + + +def _refused(error: BaseException) -> bool: + """A refusal by the binding itself, at the top of the case.""" + return type(error) is ContractCallRefused and error.code == UNSUPPORTED + + +def _idempotent(client: KitClient, case: KitCase, check: Idempotent) -> None: + before = check.observe(client) + envelope = client.envelope(case.operation) + first = check.call(client, envelope) + once = check.observe(client) + second = check.call(client, envelope) + twice = check.observe(client) + if once == before: + raise AssertionError("KIT_IDEMPOTENCY_NO_EFFECT: the first call changed nothing") + if twice != once: + raise AssertionError( + "KIT_IDEMPOTENCY_BROKEN: one idempotency key had its effect twice: " + f"{once!r} became {twice!r}" + ) + if second != first: + raise AssertionError( + f"KIT_IDEMPOTENCY_BROKEN: the repeated call answered {second!r}, first {first!r}" + ) + + +def _outcome(run: Callable[[], None]) -> tuple[Outcome, str]: + try: + run() + except KeyboardInterrupt, SystemExit: + raise + except BaseException as error: # noqa: BLE001 - a failure or a skip: the kit has no skips + outcome: Outcome = "refused" if _refused(error) else "failed" + return outcome, f"{type(error).__name__}: {error}" + return "passed", "" + + +def run_case( + kit: Kit, + case: KitCase, + make: Callable[[], object], + serialized: bool, + consumer: Version, + provided: Version, +) -> CaseResult: + """One case, here, on a fresh implementation.""" + bound = Bound(kit.contract, make(), serialized=serialized, provided=provided) + client = KitClient(bound, consumer, case.id) + check = case.check + if isinstance(check, Idempotent): + outcome, detail = _outcome(functools.partial(_idempotent, client, case, check)) + else: + outcome, detail = _outcome(functools.partial(check, client)) + return CaseResult(case.id, outcome, detail) diff --git a/packages/contracts/src/orvex/contracts/kit_process.py b/packages/contracts/src/orvex/contracts/kit_process.py new file mode 100644 index 00000000..0b96bbbb --- /dev/null +++ b/packages/contracts/src/orvex/contracts/kit_process.py @@ -0,0 +1,180 @@ +"""Running one kit case in a process of its own, built from the case's declared environment +(#486 G1): its variables and its extension directory, and nothing inherited from the kit's +process, so no case patches an implementation's internals to reach a condition. + + python -m orvex.contracts.kit_process # reads one request on stdin + +The case and its kit are found again in the child by the module that declares the case's check, +and the implementation by its factory's module and name, so both must be defined at module +level (KIT_CASE_NOT_IMPORTABLE otherwise). The child runs the case through the binding it is +asked for and sends its `CaseResult` back through the serializing codec, in a file of the case's +own directory; a child that fails, answers anything else or takes longer than `TIMEOUT` fails +the case (KIT_CASE_PROCESS_FAILED). +""" + +from __future__ import annotations + +import importlib +import json +import os +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Callable +from pathlib import Path + +from .codec import Codec +from .interface_errors import InterfaceContractError +from .kit import CaseEnvironment, Idempotent, Kit, KitCase +from .kit_case import CaseResult, run_case +from .version import Version + +TIMEOUT = 300 +CODEC = Codec(frozenset({CaseResult})) + + +def _reference(value: object) -> str: + """`module:qualified.name` of a module-level class or function.""" + name = getattr(value, "__qualname__", "") + if "<" in name or not name: + raise ValueError(f"{value!r} is not defined at module level") + return f"{value.__module__}:{name}" + + +def _resolve(reference: str) -> object: + module, _, name = reference.partition(":") + found: object = importlib.import_module(module) + for part in name.split("."): + found = getattr(found, part) + return found + + +def _root(module: str) -> str: + """The directory a module is imported from by its name, whatever made it importable here + (pytest's importlib mode imports test modules without adding their root to `sys.path`).""" + found = sys.modules[module] + path = Path(str(found.__file__)) + return str(path.parents[module.count(".") + (path.name == "__init__.py")]) + + +def _declaring_module(case: KitCase) -> str: + check = case.check + function = check.call if isinstance(check, Idempotent) else check + return str(function.__module__) + + +def _variables(environment: CaseEnvironment, directory: Path) -> dict[str, str]: + """The child's whole environment: the declared variables and the extension directory.""" + variables = dict(environment.variables) + extensions = environment.extensions + if extensions is not None: + target = directory / "extensions" + source = None if extensions.source is None else _source(extensions.source) + if source is None: + target.mkdir() + else: + shutil.copytree(source, target, ignore=_without(extensions.without)) + variables[extensions.variable] = str(target) + return variables + + +def _source(variable: str) -> Path: + value = os.environ.get(variable) + if not value or not Path(value).is_dir(): + raise ValueError(f"the extension source {variable} names no directory") + return Path(value) + + +def _without(names: tuple[str, ...]) -> Callable[[str, list[str]], set[str]]: + """Ignores every file of the named extensions (`.`), at any depth.""" + + def ignored(directory: str, entries: list[str]) -> set[str]: + return {entry for entry in entries if entry.partition(".")[0] in names} + + return ignored + + +def run_case_in_its_own_process( + kit: Kit, + case: KitCase, + make: Callable[[], object], + serialized: bool, + consumer: Version, + provided: Version, +) -> CaseResult: + """One case, in a process built from its declared environment.""" + environment = case.environment or CaseEnvironment() + try: + module, factory = _declaring_module(case), _reference(make) + roots = [_root(module), _root(factory.partition(":")[0])] + request = { + "path": [*roots, *(entry for entry in sys.path if entry)], + "module": module, + "contract": kit.contract.name, + "case": case.id, + "factory": factory, + "serialized": serialized, + "consumer": str(consumer), + "provided": str(provided), + "result": "", + } + except (ValueError, KeyError, TypeError) as error: + return CaseResult(case.id, "failed", f"KIT_CASE_NOT_IMPORTABLE: {error}") + with tempfile.TemporaryDirectory(prefix="kit-case-") as directory: + try: + variables = _variables(environment, Path(directory)) + except (OSError, ValueError) as error: + return CaseResult(case.id, "failed", f"KIT_CASE_PROCESS_FAILED: {error}") + answer = Path(directory) / "result" + request["result"] = str(answer) + try: + done = subprocess.run( + [sys.executable, "-m", __name__], + input=json.dumps(request), + env=variables, + cwd=directory, + capture_output=True, + text=True, + timeout=TIMEOUT, + check=False, + ) + except subprocess.TimeoutExpired: + return CaseResult(case.id, "failed", f"KIT_CASE_PROCESS_FAILED: over {TIMEOUT} s") + try: + result = CODEC.decode(answer.read_bytes()) + except OSError, InterfaceContractError: + result = None + if done.returncode != 0 or not isinstance(result, CaseResult) or result.id != case.id: + tail = (done.stderr or done.stdout)[-2000:] + return CaseResult(case.id, "failed", f"KIT_CASE_PROCESS_FAILED: {tail}") + return result + + +def main() -> int: + request = json.loads(sys.stdin.read()) + sys.path[:0] = [entry for entry in request["path"] if entry not in sys.path] + module = importlib.import_module(request["module"]) + kit = next( + value + for value in vars(module).values() + if isinstance(value, Kit) and value.contract.name == request["contract"] + ) + case = next(case for case in kit.cases if case.id == request["case"]) + make = _resolve(request["factory"]) + if not callable(make): + raise TypeError(f"{request['factory']} is not a factory") + result = run_case( + kit, + case, + make, + bool(request["serialized"]), + Version.parse(request["consumer"]), + Version.parse(request["provided"]), + ) + Path(request["result"]).write_bytes(CODEC.encode(result)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/contracts/src/orvex/contracts/kit_run.py b/packages/contracts/src/orvex/contracts/kit_run.py index cda34037..fe951d6a 100644 --- a/packages/contracts/src/orvex/contracts/kit_run.py +++ b/packages/contracts/src/orvex/contracts/kit_run.py @@ -1,8 +1,10 @@ """Running a conformance kit (#486 G1, G3, G5): every case, each on a fresh implementation, in process and serialized, by consumers at N-1 and N, with nothing skipped. -A run serves the implementation at the version it provides (`provided`) to a consumer at -version `consumer`, which runs the cases that exist at its version. A case that exists at the +A case that declares its own environment runs in its own process built from it +(`kit_process`); every other case runs here (`kit_case`). A run serves the implementation at +the version it provides (`provided`) to a consumer at version `consumer`, which runs the cases +that exist at its version. A case that exists at the provided version must pass. A newer case (a consumer at N against a provider at N-1) must pass or be refused with CONTRACT_VERSION_UNSUPPORTED: never a wrong answer. Each run also sends the envelope cases the contract implies, which no implementation can pass by itself: a call whose @@ -13,33 +15,19 @@ from __future__ import annotations -import functools from collections.abc import Callable, Iterator from dataclasses import dataclass -from typing import Literal from .binding import Bound from .envelope import IDEMPOTENCY_KEY, REQUIRED from .interface import InterfaceContract, Operation -from .interface_errors import ContractCallRefused, InterfaceContractError -from .kit import Idempotent, Kit, KitCase +from .interface_errors import InterfaceContractError +from .kit import Kit +from .kit_case import UNSUPPORTED, CaseResult, run_case from .kit_client import KitClient +from .kit_process import run_case_in_its_own_process from .version import Version -Outcome = Literal["passed", "refused", "failed"] -UNSUPPORTED = "CONTRACT_VERSION_UNSUPPORTED" - - -@dataclass(frozen=True, slots=True) -class CaseResult: - id: str - outcome: Outcome - detail: str - - @property - def passed(self) -> bool: - return self.outcome == "passed" - @dataclass(frozen=True, slots=True) class KitReport: @@ -68,43 +56,6 @@ def binding_name(report: KitReport) -> str: return f"{binding}, consumer {report.consumer}, provider {report.provided}" -def _refused(error: BaseException) -> bool: - """A refusal by the binding itself: never one found down a `__cause__` chain, and never one - an implementation raised, which the binding turns into INTERFACE_ERROR_UNDECLARED.""" - return type(error) is ContractCallRefused and error.code == UNSUPPORTED - - -def _idempotent(client: KitClient, case: KitCase, check: Idempotent) -> None: - before = check.observe(client) - envelope = client.envelope(case.operation) - first = check.call(client, envelope) - once = check.observe(client) - second = check.call(client, envelope) - twice = check.observe(client) - if once == before: - raise AssertionError("KIT_IDEMPOTENCY_NO_EFFECT: the first call changed nothing") - if twice != once: - raise AssertionError( - "KIT_IDEMPOTENCY_BROKEN: one idempotency key had its effect twice: " - f"{once!r} became {twice!r}" - ) - if second != first: - raise AssertionError( - f"KIT_IDEMPOTENCY_BROKEN: the repeated call answered {second!r}, first {first!r}" - ) - - -def _outcome(run: Callable[[], None]) -> tuple[Outcome, str]: - try: - run() - except KeyboardInterrupt, SystemExit: - raise - except BaseException as error: # noqa: BLE001 - a failure or a skip: the kit has no skips - outcome: Outcome = "refused" if _refused(error) else "failed" - return outcome, f"{type(error).__name__}: {error}" - return "passed", "" - - def _envelope_changes( contract: InterfaceContract, operation: Operation, consumer: Version ) -> Iterator[tuple[str, str, dict[str, str | None]]]: @@ -174,14 +125,8 @@ def run_kit( for case in kit.cases: if not speaking.provides(kit.since(case)): continue - bound = Bound(kit.contract, make(), serialized=serialized, provided=served) - client = KitClient(bound, speaking, case.id) - check = case.check - if isinstance(check, Idempotent): - outcome, detail = _outcome(functools.partial(_idempotent, client, case, check)) - else: - outcome, detail = _outcome(functools.partial(check, client)) - results.append(CaseResult(case.id, outcome, detail)) + run = run_case if case.environment is None else run_case_in_its_own_process + results.append(run(kit, case, make, serialized, speaking, served)) envelope = _envelope_results(kit, make, serialized, speaking, served) return KitReport( kit.contract.name, serialized, speaking, served, tuple(results), tuple(envelope) diff --git a/packages/contracts/tests/environment_contract.py b/packages/contracts/tests/environment_contract.py new file mode 100644 index 00000000..a9776be6 --- /dev/null +++ b/packages/contracts/tests/environment_contract.py @@ -0,0 +1,106 @@ +"""A small interface contract whose kit cases declare their own environment, and its +implementation, which reads only that environment, for the tests of the case process.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Protocol + +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation +from orvex.contracts.kit import CaseEnvironment, ExtensionDirectory, Kit, KitCase +from orvex.contracts.kit_client import KitClient +from orvex.contracts.version import Version + +ENGINE = "STATION_ENGINE_VERSION" +EXTENSIONS = "STATION_EXTENSIONS" +SOURCE = "STATION_EXTENSION_SOURCE" + + +class StationError(Exception): + def __init__(self, code: str, message: str, details: dict[str, str]) -> None: + super().__init__(f"{code}: {message}") + self.code = code + self.message = message + self.details = details + + +class Station(Protocol): + def setting(self, envelope: CallEnvelope, name: str) -> str | None: ... + + def extensions(self, envelope: CallEnvelope) -> tuple[str, ...]: ... + + +STATION = InterfaceContract( + name="station", + version=Version(1, 0), + protocol=Station, + error_type=StationError, + capabilities=("read",), + errors=(), + operations=(Operation("setting", "read"), Operation("extensions", "read")), +) + + +class EnvironmentStation: + """Answers from its process's environment and nothing else.""" + + def setting(self, envelope: CallEnvelope, name: str) -> str | None: + return os.environ.get(name) + + def extensions(self, envelope: CallEnvelope) -> tuple[str, ...]: + directory = os.environ.get(EXTENSIONS) + if directory is None: + return () + return tuple(sorted(p.name for p in Path(directory).rglob("*") if p.is_file())) + + +def _expect(condition: bool, text: str) -> None: + if not condition: + raise AssertionError(text) + + +def declared_engine(station: KitClient) -> None: + _expect(station.setting(ENGINE) == "26.10", "the declared engine version is not seen") + _expect(station.setting(SOURCE) is None, "the case inherited the kit's environment") + + +def without_httpfs(station: KitClient) -> None: + found = station.extensions() + _expect(found == ("json.duckdb_extension",), f"extensions {found}") + + +def empty_directory(station: KitClient) -> None: + _expect(station.extensions() == (), "an empty extension directory holds files") + + +STATION_KIT = Kit( + contract=STATION, + recorded=3, + cases=( + KitCase( + "setting.declared-engine", + "setting", + "the engine version the case declares is the one the implementation sees", + declared_engine, + environment=CaseEnvironment(variables=((ENGINE, "26.10"),)), + ), + KitCase( + "extensions.without-httpfs", + "extensions", + "an extension directory copied without httpfs lacks it", + without_httpfs, + environment=CaseEnvironment( + extensions=ExtensionDirectory(EXTENSIONS, source=SOURCE, without=("httpfs",)) + ), + ), + KitCase( + "extensions.empty", + "extensions", + "an empty extension directory holds no extension", + empty_directory, + environment=CaseEnvironment(extensions=ExtensionDirectory(EXTENSIONS)), + ), + ), +) diff --git a/packages/contracts/tests/test_kit.py b/packages/contracts/tests/test_kit.py index fc24d884..d19aa25b 100644 --- a/packages/contracts/tests/test_kit.py +++ b/packages/contracts/tests/test_kit.py @@ -303,6 +303,7 @@ def test_the_description_lists_every_case_and_the_recorded_count() -> None: "expectation": LEDGER_KIT.cases[4].expectation, "idempotency": True, "since": "1.0", + "environment": None, } assert cases["count.readings"]["since"] == "1.1" assert cases["names.sorted"]["idempotency"] is False diff --git a/packages/contracts/tests/test_kit_process.py b/packages/contracts/tests/test_kit_process.py new file mode 100644 index 00000000..6bb2d3b8 --- /dev/null +++ b/packages/contracts/tests/test_kit_process.py @@ -0,0 +1,107 @@ +"""A kit case with its own environment runs in a process built from that declaration alone, and +its result returns through the serializing codec (#486 G1).""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest +from orvex.contracts import kit_run +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.kit import CaseEnvironment, Kit +from orvex.contracts.kit_case import run_case +from orvex.contracts.kit_process import run_case_in_its_own_process +from orvex.contracts.kit_run import run_kit, verify_kit +from orvex.contracts.version import Version + +from .environment_contract import SOURCE, STATION, STATION_KIT, EnvironmentStation + +V10 = Version(1, 0) + + +@pytest.fixture(autouse=True) +def extension_source(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The kit's own extension directory: httpfs and json, one level down as DuckDB keeps them.""" + versioned = tmp_path / "v2.0.0" / "linux_amd64" + versioned.mkdir(parents=True) + for name in ("httpfs", "json"): + (versioned / f"{name}.duckdb_extension").write_bytes(b"\0") + monkeypatch.setenv(SOURCE, str(tmp_path)) + + +def test_each_case_sees_exactly_its_declared_environment_in_both_bindings() -> None: + reports = verify_kit(STATION_KIT, EnvironmentStation) + assert {report.serialized for report in reports} == {False, True} + assert all(r.passed for report in reports for r in report.results) + + +def test_a_case_whose_declared_environment_is_ignored_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(kit_run, "run_case_in_its_own_process", run_case) + report = run_kit(STATION_KIT, EnvironmentStation, serialized=True) + assert [r.id for r in report.results if not r.passed] == [ + "setting.declared-engine", + "extensions.without-httpfs", + ] + with pytest.raises(InterfaceContractError) as raised: + verify_kit(STATION_KIT, EnvironmentStation) + assert raised.value.code == "KIT_FAILED" + + +def test_a_factory_the_process_cannot_import_fails_the_case() -> None: + report = run_kit(STATION_KIT, lambda: EnvironmentStation(), serialized=False) + assert all(not r.passed for r in report.results) + assert "KIT_CASE_NOT_IMPORTABLE" in report.results[0].detail + + +class Broken: + def __init__(self) -> None: + raise RuntimeError("cannot start") + + +def test_a_process_that_fails_fails_the_case() -> None: + case = STATION_KIT.cases[0] + result = run_case_in_its_own_process(STATION_KIT, case, Broken, True, V10, V10) + assert result.outcome == "failed" + assert "KIT_CASE_PROCESS_FAILED" in result.detail + assert "cannot start" in result.detail + + +def test_a_missing_extension_source_fails_the_case(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(SOURCE) + report = run_kit(STATION_KIT, EnvironmentStation, serialized=False) + failed = {r.id: r.detail for r in report.results if not r.passed} + assert list(failed) == ["extensions.without-httpfs"] + assert "KIT_CASE_PROCESS_FAILED" in failed["extensions.without-httpfs"] + + +def test_the_environment_is_part_of_the_kit_s_description() -> None: + cases = STATION_KIT.describe()["cases"] + assert isinstance(cases, dict) + assert cases["setting.declared-engine"]["environment"] == { + "variables": {"STATION_ENGINE_VERSION": "26.10"}, + "extensions": None, + } + assert cases["extensions.without-httpfs"]["environment"]["extensions"] == { + "variable": "STATION_EXTENSIONS", + "source": SOURCE, + "without": ["httpfs"], + } + + +@pytest.mark.parametrize( + "variables", + [(("lower", "1"),), (("A", "1"), ("A", "2")), (("STATION_EXTENSIONS", "x"),)], + ids=["lowercase", "twice", "clashes-with-the-directory"], +) +def test_an_ill_formed_environment_is_refused(variables: tuple[tuple[str, str], ...]) -> None: + case = STATION_KIT.cases[2] + environment = CaseEnvironment( + variables, case.environment.extensions if case.environment else None + ) + changed = dataclasses.replace(case, environment=environment) + with pytest.raises(InterfaceContractError) as raised: + Kit(STATION, recorded=3, cases=(*STATION_KIT.cases[:2], changed)) + assert raised.value.code == "KIT_INVALID" From 7b871a89abd77af074693202c5110857f51292a9 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:11:33 -0400 Subject: [PATCH 12/21] refactor(testing): the shared test helpers become the test-only orvex-testing distribution `bounds`, `environment`, `fixtures`, `facade_interface` and `azurite` move from `tests/support` to `packages/testing` (`orvex.testing`), never shipped and depending on no module, and every test imports them from there. The boundaries check refuses them in product code and the consumer rule allows them in tests only; the product-name check knows the `orvex-testing` package; the standalone job installs its wheel beside a module whose tests declare it; the Windows job installs it for the tests it runs. Part of #486. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .github/workflows/python.yml | 2 + .../server/tests/test_commands_interface.py | 3 +- packages/server/tests/test_lake_data.py | 2 +- packages/server/tests/test_project.py | 3 +- .../store/tests/test_existing_lake_files.py | 3 +- .../store/tests/test_file_source_exactness.py | 2 +- packages/store/tests/test_gcs_azure.py | 3 +- packages/store/tests/test_lake.py | 3 +- packages/store/tests/test_lake_interface.py | 3 +- packages/store/tests/test_tokens.py | 3 +- packages/testing/LICENSE | 202 ++++++++++++++++++ packages/testing/README.md | 12 ++ packages/testing/pyproject.toml | 20 ++ .../testing/src/orvex/testing/__init__.py | 2 + .../testing/src/orvex/testing}/azurite.py | 2 +- .../testing/src/orvex/testing}/bounds.py | 0 .../testing/src/orvex/testing}/environment.py | 0 .../src/orvex/testing}/facade_interface.py | 0 .../testing/src/orvex/testing}/fixtures.py | 0 packages/testing/src/orvex/testing/py.typed | 0 .../testing/tests}/test_bounds.py | 3 +- tests/conformance/conftest.py | 3 +- tests/conformance/test_clickhouse_adapter.py | 2 +- tests/conformance/test_existing_ducklake.py | 2 +- tests/conformance/test_masked_engines.py | 2 +- tests/conformance/test_semantic_corpus.py | 2 +- tests/conformance/test_withheld_rows.py | 2 +- tests/tools/test_checks.py | 26 +++ tests/tools/test_consumer_boundaries.py | 29 ++- tests/tools/test_duckdb_extensions.py | 2 +- tests/tools/test_fixtures.py | 2 +- tests/tools/test_load_baseline.py | 2 +- tests/tools/test_synthetic.py | 2 +- tools/checks/boundaries.py | 24 ++- tools/checks/consumers.py | 12 +- tools/checks/product_name.py | 4 +- tools/ci/checks.toml | 2 +- tools/ci/standalone.py | 16 +- tools/repository.py | 3 + 39 files changed, 359 insertions(+), 46 deletions(-) create mode 100644 packages/testing/LICENSE create mode 100644 packages/testing/README.md create mode 100644 packages/testing/pyproject.toml create mode 100644 packages/testing/src/orvex/testing/__init__.py rename {tests/support => packages/testing/src/orvex/testing}/azurite.py (98%) rename {tests/support => packages/testing/src/orvex/testing}/bounds.py (100%) rename {tests/support => packages/testing/src/orvex/testing}/environment.py (100%) rename {tests/support => packages/testing/src/orvex/testing}/facade_interface.py (100%) rename {tests/support => packages/testing/src/orvex/testing}/fixtures.py (100%) create mode 100644 packages/testing/src/orvex/testing/py.typed rename {tests/tools => packages/testing/tests}/test_bounds.py (97%) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 33ba6531..89f2368e 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -382,6 +382,8 @@ jobs: python -m pip install --quiet pip==26.3 pins=$(python -m tools.ci.pins pylock.toml pytest pluggy iniconfig packaging pygments duckdb) python -m pip install --quiet --only-binary=:all: --no-deps $pins colorama==0.4.6 + # The shared test helpers these tests use (#486), from the checkout. + python -m pip install --quiet --no-deps -e packages/testing - run: python -m pytest -p tools.ci.order tests/tools/test_fixtures.py tests/tools/test_order.py tests/tools/test_shard.py tests/tools/test_pins.py tests/tools/test_synthetic.py env: PYTEST_RANDOM_ORDER: ${{ github.run_id }} diff --git a/packages/server/tests/test_commands_interface.py b/packages/server/tests/test_commands_interface.py index 127a2bd8..112ad274 100644 --- a/packages/server/tests/test_commands_interface.py +++ b/packages/server/tests/test_commands_interface.py @@ -18,8 +18,7 @@ from pathlib import Path import orvex.server.commands as facade - -from tests.support.facade_interface import interface, internal +from orvex.testing.facade_interface import interface, internal RECORD = Path(__file__).with_name("fixtures") / "commands_interface.json" diff --git a/packages/server/tests/test_lake_data.py b/packages/server/tests/test_lake_data.py index 2db23394..240a4554 100644 --- a/packages/server/tests/test_lake_data.py +++ b/packages/server/tests/test_lake_data.py @@ -25,7 +25,7 @@ from orvex.server.project import load_project from orvex.store.lake import Limits, open_lake -from tests.support import azurite +from orvex.testing import azurite SLUG = load_identity().slug PROJECT = "version: 1\nname: Test\ntime_zone: UTC\n" diff --git a/packages/server/tests/test_project.py b/packages/server/tests/test_project.py index 4d6c1887..384c5a75 100644 --- a/packages/server/tests/test_project.py +++ b/packages/server/tests/test_project.py @@ -18,8 +18,7 @@ ) from orvex.server.cli import main from orvex.server.project import ProjectLoadError, load_project, resolve_secret - -from tests.support.bounds import linear +from orvex.testing.bounds import linear PROJECT = """\ version: 1 diff --git a/packages/store/tests/test_existing_lake_files.py b/packages/store/tests/test_existing_lake_files.py index 93b124da..2d8bbd8d 100644 --- a/packages/store/tests/test_existing_lake_files.py +++ b/packages/store/tests/test_existing_lake_files.py @@ -33,8 +33,7 @@ from orvex.store import separation from orvex.store.errors import StoreError from orvex.store.existing_lake import ExistingLake - -from tests.support.environment import unavailable +from orvex.testing.environment import unavailable MEMORY = "512MB" diff --git a/packages/store/tests/test_file_source_exactness.py b/packages/store/tests/test_file_source_exactness.py index 69c99b07..94d94d5f 100644 --- a/packages/store/tests/test_file_source_exactness.py +++ b/packages/store/tests/test_file_source_exactness.py @@ -23,9 +23,9 @@ import pytest from orvex.store.errors import StoreError from orvex.store.file_sources import view_query +from orvex.testing.fixtures import frozen from tests.support import parquet_footer -from tests.support.fixtures import frozen D = decimal.Decimal UTC = dt.UTC diff --git a/packages/store/tests/test_gcs_azure.py b/packages/store/tests/test_gcs_azure.py index 64477d44..930a3f60 100644 --- a/packages/store/tests/test_gcs_azure.py +++ b/packages/store/tests/test_gcs_azure.py @@ -30,8 +30,7 @@ ) from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake - -from tests.support import azurite +from orvex.testing import azurite LIMITS = Limits( memory_limit="512MB", threads=1, max_concurrent=4, queue_size=16, default_timeout_ms=600_000 diff --git a/packages/store/tests/test_lake.py b/packages/store/tests/test_lake.py index 2aced6d6..c8a7186b 100644 --- a/packages/store/tests/test_lake.py +++ b/packages/store/tests/test_lake.py @@ -16,8 +16,7 @@ from orvex.store.errors import StoreError from orvex.store.json_encoding import arrow_to_json from orvex.store.lake import FOLDER_COST, PATTERN_BUDGET, Lake, Limits, open_lake - -from tests.support.bounds import SAFETY_NET, FolderReads +from orvex.testing.bounds import SAFETY_NET, FolderReads # The shared contract tests run on every catalog, SQLite included (#120). EVERY_CATALOG = True diff --git a/packages/store/tests/test_lake_interface.py b/packages/store/tests/test_lake_interface.py index 82d15ed0..d0f41e2b 100644 --- a/packages/store/tests/test_lake_interface.py +++ b/packages/store/tests/test_lake_interface.py @@ -15,8 +15,7 @@ from pathlib import Path import orvex.store.lake as facade - -from tests.support.facade_interface import interface, internal +from orvex.testing.facade_interface import interface, internal RECORD = Path(__file__).with_name("fixtures") / "lake_interface.json" diff --git a/packages/store/tests/test_tokens.py b/packages/store/tests/test_tokens.py index 3eebc6eb..85b0ff4a 100644 --- a/packages/store/tests/test_tokens.py +++ b/packages/store/tests/test_tokens.py @@ -24,8 +24,7 @@ _single_statement, ) from orvex.store.sql_parser import PARSER_SPACES, _plain, _tokens - -from tests.support.bounds import FACTOR, SAFETY_NET, FolderReads, linear +from orvex.testing.bounds import FACTOR, SAFETY_NET, FolderReads, linear def within_budget(reads: FolderReads) -> None: diff --git a/packages/testing/LICENSE b/packages/testing/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/packages/testing/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/testing/README.md b/packages/testing/README.md new file mode 100644 index 00000000..4ca425c4 --- /dev/null +++ b/packages/testing/README.md @@ -0,0 +1,12 @@ +# `orvex-testing` — `orvex.testing` + +Test helpers that no single module owns, shared by the modules' own tests and the repository's +tests (#486 §2 item 4). It is never shipped, depends on no Orvex module, and is installed beside a +module in its standalone job. Only tests may import it; the boundaries check refuses it anywhere +else. A helper that only one module's tests use lives in that module's tests instead. + +- `bounds`: cost bounds that fail a quadratic regression, not a slow runner (#329). +- `environment`: a test that needs a service skips locally and fails in CI (`unavailable`). +- `fixtures`: shared fixture files frozen read-only, and private copies (#376). +- `facade_interface`: the recorded names and shapes of a module's façade (#246). +- `azurite`: a client of the Azurite blob emulator for object-storage tests (#127). diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml new file mode 100644 index 00000000..565387c4 --- /dev/null +++ b/packages/testing/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools==84.2.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "orvex-testing" +version = "0.3.0" +description = "Test helpers shared by the modules' and the repository's tests; never shipped." +readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.14" +dependencies = ["pytest==9.2.0"] + +[tool.setuptools.packages.find] +where = ["src"] +namespaces = true + +[tool.setuptools.package-data] +"orvex.testing" = ["py.typed"] diff --git a/packages/testing/src/orvex/testing/__init__.py b/packages/testing/src/orvex/testing/__init__.py new file mode 100644 index 00000000..8acb8397 --- /dev/null +++ b/packages/testing/src/orvex/testing/__init__.py @@ -0,0 +1,2 @@ +"""Test helpers shared by the modules' and the repository's tests (#486 §2 item 4): never +shipped, importable only from tests, and depending on no module of the product.""" diff --git a/tests/support/azurite.py b/packages/testing/src/orvex/testing/azurite.py similarity index 98% rename from tests/support/azurite.py rename to packages/testing/src/orvex/testing/azurite.py index 0a62dccf..1def792b 100644 --- a/tests/support/azurite.py +++ b/packages/testing/src/orvex/testing/azurite.py @@ -15,7 +15,7 @@ import urllib.request import xml.etree.ElementTree as ElementTree -from tests.support.environment import unavailable +from orvex.testing.environment import unavailable VARIABLE = "STORE_TEST_AZURE_CONNECTION_STRING" VERSION = "2021-08-06" diff --git a/tests/support/bounds.py b/packages/testing/src/orvex/testing/bounds.py similarity index 100% rename from tests/support/bounds.py rename to packages/testing/src/orvex/testing/bounds.py diff --git a/tests/support/environment.py b/packages/testing/src/orvex/testing/environment.py similarity index 100% rename from tests/support/environment.py rename to packages/testing/src/orvex/testing/environment.py diff --git a/tests/support/facade_interface.py b/packages/testing/src/orvex/testing/facade_interface.py similarity index 100% rename from tests/support/facade_interface.py rename to packages/testing/src/orvex/testing/facade_interface.py diff --git a/tests/support/fixtures.py b/packages/testing/src/orvex/testing/fixtures.py similarity index 100% rename from tests/support/fixtures.py rename to packages/testing/src/orvex/testing/fixtures.py diff --git a/packages/testing/src/orvex/testing/py.typed b/packages/testing/src/orvex/testing/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/tests/tools/test_bounds.py b/packages/testing/tests/test_bounds.py similarity index 97% rename from tests/tools/test_bounds.py rename to packages/testing/tests/test_bounds.py index 14471a06..96202b69 100644 --- a/tests/tools/test_bounds.py +++ b/packages/testing/tests/test_bounds.py @@ -7,8 +7,7 @@ from pathlib import Path import pytest - -from tests.support import bounds +from orvex.testing import bounds def test_linear_work_passes_and_quadratic_work_fails() -> None: diff --git a/tests/conformance/conftest.py b/tests/conformance/conftest.py index d0794ac7..a246d919 100644 --- a/tests/conformance/conftest.py +++ b/tests/conformance/conftest.py @@ -8,8 +8,7 @@ from __future__ import annotations import pytest - -from tests.support.environment import unavailable +from orvex.testing.environment import unavailable @pytest.fixture(autouse=True) diff --git a/tests/conformance/test_clickhouse_adapter.py b/tests/conformance/test_clickhouse_adapter.py index d0852224..a3c3c854 100644 --- a/tests/conformance/test_clickhouse_adapter.py +++ b/tests/conformance/test_clickhouse_adapter.py @@ -55,6 +55,7 @@ from orvex.semantic.result.descriptors import arrow_type from orvex.store.errors import StoreError from orvex.store.lake import Lake +from orvex.testing.environment import unavailable from conformance.runners import bi_platform_adapter from conformance.runners import corpus_execution as ce @@ -63,7 +64,6 @@ from tests.conformance import test_row_policies as rp from tests.conformance.test_execution import no_policies from tests.properties import exact as generated_exact -from tests.support.environment import unavailable from tests.support.policies import VENDORS, extension Document = dict[str, Any] diff --git a/tests/conformance/test_existing_ducklake.py b/tests/conformance/test_existing_ducklake.py index 2bf5490a..89279aa3 100644 --- a/tests/conformance/test_existing_ducklake.py +++ b/tests/conformance/test_existing_ducklake.py @@ -41,12 +41,12 @@ from orvex.store.errors import StoreError from orvex.store.existing_lake import ExistingLake from orvex.store.existing_lake_reads import LakeReads +from orvex.testing.environment import unavailable from conformance.runners import corpus_execution as ce from tests.conformance import test_row_policies as rp from tests.conformance.test_direct_query import CASES, ZONE, routed from tests.conformance.test_execution import ask -from tests.support.environment import unavailable from tests.support.policies import VENDORS, extension ROOT = Path(__file__).resolve().parents[2] diff --git a/tests/conformance/test_masked_engines.py b/tests/conformance/test_masked_engines.py index ca7b34fe..63e9e082 100644 --- a/tests/conformance/test_masked_engines.py +++ b/tests/conformance/test_masked_engines.py @@ -20,12 +20,12 @@ import pytest import yaml +from orvex.testing.fixtures import frozen from conformance.runners import corpus_execution as ce from conformance.runners import security_fixtures as sf from conformance.runners.security_cases import SEPARATOR from conformance.runners.security_engines import ask, served -from tests.support.fixtures import frozen BASE = sf.fixture("direct-query", ce.VENDORS[0]) OUTAGES: list[list[str]] = [[str(v) for v in r] for r in BASE.rows["outages"]] diff --git a/tests/conformance/test_semantic_corpus.py b/tests/conformance/test_semantic_corpus.py index 15d7db25..d0b27c47 100644 --- a/tests/conformance/test_semantic_corpus.py +++ b/tests/conformance/test_semantic_corpus.py @@ -18,12 +18,12 @@ from typing import Any import pytest +from orvex.testing.environment import unavailable from conformance.runners import bi_platform_adapter as adapter from conformance.runners import corpus_execution as ce from conformance.runners import semantic_corpus from conformance.runners.semantic_corpus import CORPUS, OSSIE, ROOT, check_vendored, run -from tests.support.environment import unavailable def copy_repository(tmp_path: Path) -> Path: diff --git a/tests/conformance/test_withheld_rows.py b/tests/conformance/test_withheld_rows.py index 5b028f56..2f87fb48 100644 --- a/tests/conformance/test_withheld_rows.py +++ b/tests/conformance/test_withheld_rows.py @@ -27,11 +27,11 @@ from orvex.semantic.execution import answer from orvex.semantic.ossie.reader import read from orvex.semantic.policies import read_policies +from orvex.testing.fixtures import frozen from conformance.runners import corpus_execution as ce from conformance.runners import security_fixtures as sf from conformance.runners.security_engines import ask, served -from tests.support.fixtures import frozen BASE = sf.fixture("direct-query", ce.VENDORS[0]) OWN_ZONE = {"name": "own_zone", "dataset": "outages", "kind": "predicate", diff --git a/tests/tools/test_checks.py b/tests/tools/test_checks.py index 53f85251..26287be1 100644 --- a/tests/tools/test_checks.py +++ b/tests/tools/test_checks.py @@ -103,6 +103,32 @@ def test_server_may_import_every_module(self) -> None: tree = parse("import orvex.semantic\nfrom orvex.store import x\n") assert boundary_findings(tree, "server", "orvex", "packages/server/src/a.py") == [] + @pytest.mark.parametrize( + ("owner", "relative"), + [ + ("store", "packages/store/src/orvex/store/a.py"), + ("server", "packages/server/src/orvex/server/a.py"), + ("contracts", "packages/contracts/src/orvex/contracts/a.py"), + ], + ) + def test_product_code_importing_the_test_helpers_is_reported( + self, owner: str, relative: str + ) -> None: + # #486 §2 item 4: `orvex.testing` is never shipped, so no product code may need it. + tree = parse("from orvex.testing.bounds import linear\n") + findings = boundary_findings(tree, owner, "orvex", relative) + assert [f.code for f in findings] == ["MODULE_BOUNDARY_VIOLATION"] + + def test_a_module_s_tests_may_import_the_test_helpers(self) -> None: + tree = parse("from orvex.testing import environment\n") + assert boundary_findings(tree, "store", "orvex", "packages/store/tests/t.py") == [] + + def test_the_test_helpers_may_import_no_module_but_the_contracts(self) -> None: + tree = parse("import orvex.store.lake\nimport orvex.contracts.store\n") + relative = "packages/testing/src/orvex/testing/a.py" + findings = boundary_findings(tree, "testing", "orvex", relative) + assert [f.detail for f in findings] == ["module 'testing' imports 'orvex.store'"] + def test_own_module_contracts_and_third_party_are_allowed(self) -> None: tree = parse("import orvex.store.core\nimport orvex.contracts.x\nimport duckdb\n") assert boundary_findings(tree, "store", "orvex", "f.py") == [] diff --git a/tests/tools/test_consumer_boundaries.py b/tests/tools/test_consumer_boundaries.py index 56d9416f..cf365809 100644 --- a/tests/tools/test_consumer_boundaries.py +++ b/tests/tools/test_consumer_boundaries.py @@ -29,6 +29,8 @@ "dunder": ("tools/y.py", '__import__("orvex.mcp.server")\n'), "api outside assembly": ("packages/server/src/orvex/server/y.py", "import orvex.store.api\n"), "internal in assembly": (ASSEMBLY, "from orvex.store.lake import x\n"), + "test helpers in a runner": ("conformance/runners/y.py", "import orvex.testing.environment\n"), + "test helpers in a tool": ("tools/z.py", "from orvex.testing.fixtures import frozen\n"), } @@ -56,6 +58,14 @@ def test_a_planted_internal_import_is_caught(relative: str, source: str) -> None "relative": ("packages/server/src/orvex/server/x.py", "from . import cli\n"), "third party": ("tools/x.py", "import duckdb\nfrom tools.repository import identity\n"), "computed dynamic import": ("tools/x.py", "import importlib\nimportlib.import_module(name)\n"), + "test helpers in a repository test": ( + "tests/x/t.py", + "from orvex.testing.bounds import linear\n", + ), + "test helpers in a server test": ( + "packages/server/tests/t.py", + "from orvex.testing import azurite\n", + ), } @@ -81,8 +91,8 @@ def test_the_scan_covers_the_consumers_and_leaves_the_modules_to_the_module_rule summary = consumer_summary(found) assert summary.startswith(f"consumer boundaries (report mode until #486 step 7): {len(found)}") assert "server 4" in summary - assert "runners 1" in summary - assert "tools 2" in summary + assert "runners 2" in summary + assert "tools 3" in summary assert "tests 2" in summary @@ -100,6 +110,21 @@ def test_report_mode_prints_every_violation_and_does_not_fail( assert "consumer boundaries (report mode until #486 step 7)" in out +def test_server_code_importing_the_test_helpers_fails_as_a_module_violation( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + # The server's code is product code too: the blocking module rule catches it, not only the + # consumer report. + empty = tmp_path / "empty.toml" + empty.write_text("") + files = {"packages/server/src/orvex/server/z.py": "from orvex.testing import bounds\n"} + (tmp_path / "r").mkdir() + assert run(make_repository(tmp_path / "r", files), empty, empty) == 1 + assert "MODULE_BOUNDARY_VIOLATION: packages/server/src/orvex/server/z.py:1" in ( + capsys.readouterr().out + ) + + def test_a_module_boundary_violation_still_fails(tmp_path: Path) -> None: empty = tmp_path / "empty.toml" empty.write_text("") diff --git a/tests/tools/test_duckdb_extensions.py b/tests/tools/test_duckdb_extensions.py index 9ae1c566..5499929d 100644 --- a/tests/tools/test_duckdb_extensions.py +++ b/tests/tools/test_duckdb_extensions.py @@ -10,8 +10,8 @@ import duckdb import pytest +from orvex.testing.environment import unavailable -from tests.support.environment import unavailable from tools.ci import duckdb_extensions as tool from tools.repository import REPOSITORY_ROOT diff --git a/tests/tools/test_fixtures.py b/tests/tools/test_fixtures.py index 796a70d7..a3ab1d3a 100644 --- a/tests/tools/test_fixtures.py +++ b/tests/tools/test_fixtures.py @@ -7,8 +7,8 @@ from pathlib import Path import pytest +from orvex.testing.fixtures import freeze, frozen, private_copy, thaw -from tests.support.fixtures import freeze, frozen, private_copy, thaw from tools.repository import REPOSITORY_ROOT SHARED = {"class", "module", "package", "session", "unknown"} diff --git a/tests/tools/test_load_baseline.py b/tests/tools/test_load_baseline.py index 187db9ae..9ac85de1 100644 --- a/tests/tools/test_load_baseline.py +++ b/tests/tools/test_load_baseline.py @@ -11,8 +11,8 @@ import duckdb import pytest +from orvex.testing.fixtures import frozen, private_copy -from tests.support.fixtures import frozen, private_copy from tools.benchmarks.load_baseline import ( APPENDED, Step, diff --git a/tests/tools/test_synthetic.py b/tests/tools/test_synthetic.py index 5d9b85f5..280969f0 100644 --- a/tests/tools/test_synthetic.py +++ b/tests/tools/test_synthetic.py @@ -10,8 +10,8 @@ import duckdb import pytest +from orvex.testing.fixtures import frozen -from tests.support.fixtures import frozen from tools.synthetic.generate import PRICES, TABLES, SyntheticError, generate ROOT = Path(__file__).resolve().parents[2] diff --git a/tools/checks/boundaries.py b/tools/checks/boundaries.py index 4bb2b1e7..2aff0f1c 100644 --- a/tools/checks/boundaries.py +++ b/tools/checks/boundaries.py @@ -33,7 +33,7 @@ materialization_findings, stale_entries, ) -from tools.repository import MODULES, REPOSITORY_ROOT, identity +from tools.repository import MODULES, REPOSITORY_ROOT, TESTING, identity __all__ = [ "MATERIALIZING_NAMES", @@ -58,9 +58,9 @@ def boundary_findings(tree: ast.AST, owner: str, namespace: str, relative: str) -> list[Finding]: + findings = _testing_findings(tree, owner, namespace, relative) if owner == COMPOSER: - return [] - findings: list[Finding] = [] + return findings reported: set[tuple[str, int]] = set() for name, line in imported_names(tree): parts = name.split(".") @@ -80,6 +80,24 @@ def boundary_findings(tree: ast.AST, owner: str, namespace: str, relative: str) return findings +def _testing_findings(tree: ast.AST, owner: str, namespace: str, relative: str) -> list[Finding]: + """Product code (a module's `src`) importing the test helpers, which are never shipped.""" + if "/src/" not in relative or owner == TESTING: + return [] + lines = sorted( + {line for name, line in imported_names(tree) if name.split(".")[:2] == [namespace, TESTING]} + ) + return [ + Finding( + "MODULE_BOUNDARY_VIOLATION", + relative, + line, + f"module '{owner}' imports '{namespace}.{TESTING}', the test helpers", + ) + for line in lines + ] + + def scan(root: Path, allowed: set[tuple[str, str]]) -> list[Finding]: namespace = str(identity(root)["python_namespace"]) findings: list[Finding] = [] diff --git a/tools/checks/consumers.py b/tools/checks/consumers.py index 9fc9293c..272e5653 100644 --- a/tools/checks/consumers.py +++ b/tools/checks/consumers.py @@ -1,6 +1,7 @@ """Consumer boundaries (#486 G2): `server`, the conformance runners, `tools/` and the repository tests import only `.contracts`; `server/assembly.py` may also import the modules' -`api` packages, and the server's own files the server. +`api` packages, and the server's own files the server. Tests (`tests/`, a package's `tests/`) +may also import the shared test helpers, `.testing`; nothing else may. CONSUMER_BOUNDARY_VIOLATION: any other import of a module, as `tools/checks/imports.py` reads it (static, or dynamic with a constant name). In report mode until #486 step 7: @@ -16,7 +17,7 @@ from tools.checks.architecture import Finding from tools.checks.imports import imported_names -from tools.repository import MODULES, identity +from tools.repository import MODULES, TESTING, identity CODE = "CONSUMER_BOUNDARY_VIOLATION" # Each consumer area, by the start of its paths. @@ -34,6 +35,10 @@ def assembly(namespace: str) -> str: return f"packages/server/src/{namespace}/server/assembly.py" +def _is_test(relative: str) -> bool: + return relative.startswith("tests/") or "/tests/" in relative + + def _area(relative: str) -> str | None: return next((area for area, start in AREAS.items() if relative.startswith(start)), None) @@ -47,11 +52,12 @@ def consumer_findings(tree: ast.AST, namespace: str, relative: str) -> list[Find reported: set[tuple[str, int]] = set() for name, line in imported_names(tree): parts = name.split(".") - if len(parts) < 2 or parts[0] != namespace or parts[1] not in MODULES: + if len(parts) < 2 or parts[0] != namespace or parts[1] not in (*MODULES, TESTING): continue target = parts[1] allowed = ( target == SHARED + or (target == TESTING and _is_test(relative)) or (area == "server" and target == "server") or (relative == assembly(namespace) and len(parts) > 2 and parts[2] == PUBLIC) ) diff --git a/tools/checks/product_name.py b/tools/checks/product_name.py index 0208678f..c584a6bc 100644 --- a/tools/checks/product_name.py +++ b/tools/checks/product_name.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from pathlib import Path -from tools.repository import MODULES, REPOSITORY_ROOT, identity, read_text, tracked_files +from tools.repository import MODULES, REPOSITORY_ROOT, TESTING, identity, read_text, tracked_files EXEMPT = ( "*.md", @@ -48,7 +48,7 @@ def is_exempt(relative: str) -> bool: def occurrence_pattern(name: str, slug: str) -> re.Pattern[str]: words = "|".join(re.escape(word) for word in {name, slug}) - modules = "|".join(MODULES) + modules = "|".join((*MODULES, TESTING)) # `orvex-testing`: the test helpers, a package too return re.compile( rf"(? In a temporary directory outside the checkout: copy the module's package, build its sdist and, -from that sdist, its wheel (and the contracts wheel for any other module); create a clean -virtual environment; install the wheels and the module's test dependencies, every version -constrained to the lock file of this Python; run `tools/checks/module_dependencies.py` there; +from that sdist, its wheel (and the contracts wheel for any other module, and the shared test +helpers' wheel, `orvex-testing`, when its tests declare it); create a clean virtual +environment; install the wheels and the module's test dependencies, every version constrained +to the lock file of this Python; run `tools/checks/module_dependencies.py` there; and run the module's own tests with no repository configuration. A failed step, a failed test or a skipped test (STANDALONE_TESTS_SKIPPED: a test that did not run never counts) fails the job. """ @@ -24,7 +25,7 @@ from pathlib import Path from tools.ci.pins import pins -from tools.repository import MODULES, REPOSITORY_ROOT, identity +from tools.repository import MODULES, REPOSITORY_ROOT, TESTING, identity PIP = "pip==26.3" PYTEST_CONFIG = """[pytest] @@ -108,7 +109,12 @@ def run(module: str) -> int: pinned = work / "constraints.txt" pinned.write_text("\n".join(constraints(lock_file())) + "\n", encoding="utf-8") manifest = (package / "src" / namespace / module / "module.toml").read_text("utf-8") - tests = tomllib.loads(manifest)["dependencies"]["tests"] + declared = tomllib.loads(manifest)["dependencies"]["tests"] + # The shared test helpers are never published: built from the checkout like the module. + testing = f"{slug}-{TESTING}" + if testing in declared: + wheels.append(build(REPOSITORY_ROOT / "packages" / TESTING, work)) + tests = [name for name in declared if name != testing] _step( f"install {distribution}" + ("" if module == "contracts" else f" with only {slug}-contracts"), diff --git a/tools/repository.py b/tools/repository.py index 6e99b780..ccf0ed34 100644 --- a/tools/repository.py +++ b/tools/repository.py @@ -9,6 +9,9 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] MODULES = ("store", "transform", "semantic", "access", "dashboards", "mcp", "server", "contracts") +# The test-only distribution of shared test helpers (#486 §2 item 4): never shipped, not a +# module; only tests may import it. +TESTING = "testing" _SKIPPED_DIRECTORIES = { ".git", ".venv", From b9ad7951fda3e222ec387e89bd27a4ded9eb43ba Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 02:51:25 -0400 Subject: [PATCH 13/21] test(store): the case matrix of the store's interface contract and its consumers The store's kit run in process and serialized with planted broken guarantees, the Arrow IPC transport, the codec's aliased models and fixed offsets, the ClickHouse scanner fuzzing as the store's own test, the refusal before attach, and the store's consumers' tests on the contract. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .../src/orvex/contracts/store/errors.py | 94 ++++++++++ packages/contracts/tests/test_codec_values.py | 50 +++++ .../contracts/tests/test_store_arrow_ipc.py | 29 +++ packages/server/tests/store_faults.py | 66 +++++++ .../server/tests/test_clickhouse_wiring.py | 2 +- packages/server/tests/test_commands.py | 4 +- packages/server/tests/test_evidence_runs.py | 2 +- packages/server/tests/test_existing_lakes.py | 25 +-- packages/server/tests/test_lake_catalog.py | 7 +- packages/server/tests/test_lake_data.py | 5 +- packages/server/tests/test_lake_refusals.py | 2 +- packages/server/tests/test_lake_separation.py | 2 +- packages/server/tests/test_maintenance.py | 4 +- .../server/tests/test_permission_state.py | 2 +- packages/server/tests/test_query_cancel.py | 2 +- packages/server/tests/test_run_records.py | 107 ++++++----- packages/server/tests/test_schedule.py | 2 +- .../server/tests/test_semantic_explain.py | 2 +- packages/server/tests/test_semantic_http.py | 10 +- packages/server/tests/test_semantic_model.py | 2 +- packages/server/tests/test_since.py | 2 +- packages/store/benchmarks/source_stream.py | 169 +++++++++++++++++ packages/store/tests/contract_client.py | 16 ++ .../store/tests/fixtures/lake_interface.json | 1 - .../store/tests}/parquet_footer.py | 0 packages/store/tests/test_admission_edges.py | 2 +- packages/store/tests/test_budgets.py | 2 +- packages/store/tests/test_catalog_limits.py | 2 +- packages/store/tests/test_catalog_location.py | 2 +- packages/store/tests/test_changes.py | 2 +- packages/store/tests/test_clickhouse.py | 2 +- .../tests/test_clickhouse_scanner_fuzzing.py | 172 ++++++++++++++++++ packages/store/tests/test_contract_kit.py | 91 +++++++++ packages/store/tests/test_database_sources.py | 2 +- packages/store/tests/test_existing_lake.py | 2 +- .../store/tests/test_existing_lake_files.py | 24 ++- .../store/tests/test_existing_lake_reads.py | 2 +- .../store/tests/test_file_source_exactness.py | 4 +- packages/store/tests/test_gcs_azure.py | 2 +- packages/store/tests/test_json_encoding.py | 11 +- packages/store/tests/test_lake.py | 7 +- .../store/tests/test_lake_configuration.py | 2 +- packages/store/tests/test_load_arrow.py | 2 +- packages/store/tests/test_maintenance.py | 2 +- packages/store/tests/test_nested_exactness.py | 2 +- packages/store/tests/test_object_storage.py | 2 +- packages/store/tests/test_postgres_catalog.py | 2 +- packages/store/tests/test_read_phases.py | 2 +- packages/store/tests/test_read_scopes.py | 2 +- packages/store/tests/test_redaction.py | 2 +- packages/store/tests/test_source_isolation.py | 2 +- packages/store/tests/test_source_tls.py | 2 +- packages/store/tests/test_sources.py | 2 +- packages/store/tests/test_statistics.py | 2 +- packages/store/tests/test_table_columns.py | 2 +- packages/store/tests/test_tokens.py | 2 +- tests/conformance/test_asof_windows.py | 2 +- tests/conformance/test_bridges.py | 2 +- tests/conformance/test_clickhouse_adapter.py | 4 +- .../test_clickhouse_capabilities.py | 10 +- tests/conformance/test_clickhouse_engine.py | 10 +- tests/conformance/test_conformed.py | 2 +- tests/conformance/test_detail_rows.py | 2 +- tests/conformance/test_direct_query.py | 2 +- tests/conformance/test_estimates.py | 4 +- tests/conformance/test_evidence.py | 4 +- tests/conformance/test_exact_arithmetic.py | 2 +- tests/conformance/test_execution.py | 10 +- tests/conformance/test_existing_ducklake.py | 24 +-- tests/conformance/test_explain.py | 2 +- tests/conformance/test_fanout_isolation.py | 2 +- tests/conformance/test_paths_roles.py | 2 +- tests/conformance/test_period_end.py | 6 +- tests/conformance/test_ratios.py | 2 +- tests/conformance/test_receipt_result.py | 24 ++- tests/conformance/test_result_cache.py | 4 +- tests/conformance/test_row_policies.py | 2 +- tests/conformance/test_schema_sources.py | 4 +- tests/conformance/test_semantic_result.py | 24 +-- tests/conformance/test_zones.py | 2 +- tests/properties/exact.py | 2 +- tests/properties/policy_engine.py | 2 +- tests/properties/test_engine_oracle.py | 10 +- tests/properties/test_exact_properties.py | 2 +- tests/properties/test_hidden_key_inference.py | 2 +- tests/properties/test_parser_fuzzing.py | 158 +--------------- tests/properties/world.py | 2 +- 87 files changed, 933 insertions(+), 363 deletions(-) create mode 100644 packages/contracts/src/orvex/contracts/store/errors.py create mode 100644 packages/contracts/tests/test_codec_values.py create mode 100644 packages/contracts/tests/test_store_arrow_ipc.py create mode 100644 packages/server/tests/store_faults.py create mode 100644 packages/store/benchmarks/source_stream.py create mode 100644 packages/store/tests/contract_client.py rename {tests/support => packages/store/tests}/parquet_footer.py (100%) create mode 100644 packages/store/tests/test_clickhouse_scanner_fuzzing.py create mode 100644 packages/store/tests/test_contract_kit.py diff --git a/packages/contracts/src/orvex/contracts/store/errors.py b/packages/contracts/src/orvex/contracts/store/errors.py new file mode 100644 index 00000000..a12d972c --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/errors.py @@ -0,0 +1,94 @@ +"""The store contract's error type and its builders (`contracts/store/v1`, `ErrorCode`): a +stable code, a message and details, with every registered secret redacted (#256).""" + +from __future__ import annotations + +import json +import threading +import urllib.parse + +from orvex.contracts.store.v1 import ( + ErrorCode, + ErrorDetail, + FileSourceAttachment, + SqliteSourceAttachment, +) +from orvex.contracts.store.v1 import StoreError as StoreErrorMessage + +_SECRETS: set[str] = set() +_LOCK = threading.Lock() # registration and redaction may run in several threads + + +def register_secret(value: str) -> None: + """A value that must never appear in an error: it is replaced by `***` everywhere, in every + form it takes on its way into a message (#256): as written, escaped in an SQL string (`'` + doubled) or identifier (`"` doubled), in a libpq connection string (`\\` and `\'`), in JSON + and in a URL.""" + if value: + with _LOCK: + _SECRETS.update(_forms(value)) + + +def _forms(value: str) -> set[str]: + return { + value, + value.replace("'", "''"), + value.replace('"', '""'), + value.replace("\\", "\\\\").replace("'", "\\'"), + json.dumps(value)[1:-1], + json.dumps(value, ensure_ascii=False)[1:-1], + urllib.parse.quote(value, safe=""), + urllib.parse.quote_plus(value, safe=""), + } + + +def redact(text: str) -> str: + # The longest form first, so a form inside another never leaves part of a secret behind. + with _LOCK: + secrets = sorted(_SECRETS, key=len, reverse=True) + for secret in secrets: + text = text.replace(secret, "***") + return text + + +class StoreError(Exception): + """A store failure with a stable code from the contract. Registered secrets are redacted.""" + + def __init__(self, code: str, message: str, details: dict[str, str]) -> None: + message = redact(message) + details = {name: redact(value) for name, value in details.items()} + super().__init__(f"{code}: {message}") + # Validating through the generated type rejects a code the contract does not define. + self.code: str = ErrorCode.model_validate(code).root + self.message = message + self.details = dict(details) + + def to_message(self) -> StoreErrorMessage: + return StoreErrorMessage( + kind="store_error", + code=ErrorCode.model_validate(self.code), + message=self.message, + details=[ErrorDetail(name=name, value=value) for name, value in self.details.items()], + ) + + +SNAPSHOT_CHANGED = "snapshot_changed" # `reason`: an existing lake's snapshot is no longer served +FILE_REFUSED = "file_refused" # `reason`: the sandbox refused a file an existing lake named + + +def separation_refused(name: str, reason: str) -> StoreError: + """A lake that cannot be shown apart from the instance's other files (P2-30).""" + return StoreError( + "STORE_SOURCE_INVALID", + f"Connection {name}: the lake cannot be shown apart from the instance's other files: " + f"{reason}.", + {"connection": name, "reason": reason}, + ) + + +def source_not_found(source: FileSourceAttachment | SqliteSourceAttachment) -> StoreError: + return StoreError( + "SOURCE_NOT_FOUND", + f"No file matches the path of source {source.name}.", + {"source": source.name}, + ) diff --git a/packages/contracts/tests/test_codec_values.py b/packages/contracts/tests/test_codec_values.py new file mode 100644 index 00000000..878275d5 --- /dev/null +++ b/packages/contracts/tests/test_codec_values.py @@ -0,0 +1,50 @@ +"""#486 G5: values the codec must carry exactly that the store contract showed it refused: a model +with field aliases, and an instant whose fixed offset is pydantic's own zone class.""" + +from __future__ import annotations + +import datetime as dt +from zoneinfo import ZoneInfo + +import pytest +from orvex.contracts.codec import Codec +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.store.v1 import LakeTable, Snapshot, TableColumn + + +def test_a_model_with_an_aliased_field_crosses_exactly() -> None: + column = TableColumn(name="a", type="DECIMAL(18,2)", nullable=False) + table = LakeTable(schema="main", table="t", columns=[column]) + codec = Codec(frozenset({LakeTable})) + assert codec.decode(codec.encode(table)) == table + + +def test_an_instant_parsed_by_pydantic_keeps_its_offset() -> None: + snapshot = Snapshot.model_validate( + {"kind": "snapshot", "id": 1, "committed_at": "2026-03-29T01:30:00+02:00"} + ) + assert type(snapshot.committed_at.tzinfo) is not dt.timezone # pydantic's own class + codec = Codec(frozenset({Snapshot})) + crossed = codec.decode(codec.encode(snapshot.committed_at)) + assert isinstance(crossed, dt.datetime) + assert crossed == snapshot.committed_at + assert crossed.utcoffset() == dt.timedelta(hours=2) + + +class _Shifting(dt.tzinfo): + """A zone whose offset depends on the instant: not a fixed offset, and not an IANA key.""" + + def utcoffset(self, moment: dt.datetime | None) -> dt.timedelta: + return dt.timedelta(hours=1 if moment is None or moment.month < 6 else 2) + + def dst(self, moment: dt.datetime | None) -> dt.timedelta: + return dt.timedelta(0) + + +def test_a_zone_that_is_neither_fixed_nor_named_is_refused() -> None: + codec = Codec(frozenset()) + with pytest.raises(InterfaceContractError) as caught: + codec.encode(dt.datetime(2026, 7, 1, tzinfo=_Shifting())) + assert caught.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" + named = dt.datetime(2026, 7, 1, tzinfo=ZoneInfo("America/Lima")) + assert codec.decode(codec.encode(named)) == named diff --git a/packages/contracts/tests/test_store_arrow_ipc.py b/packages/contracts/tests/test_store_arrow_ipc.py new file mode 100644 index 00000000..0c848033 --- /dev/null +++ b/packages/contracts/tests/test_store_arrow_ipc.py @@ -0,0 +1,29 @@ +"""#486 G5: Arrow crosses the store's contract as IPC streams, exactly and batch by batch.""" + +from __future__ import annotations + +from decimal import Decimal + +import pyarrow as pa +from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch, stream_bytes, stream_reader + + +def test_ipc_carries_arrow_exactly_and_lazily() -> None: + metadata = {b"duckdb_type": b"DECIMAL(38,10)"} + field = pa.field("amount", pa.decimal128(38, 10), metadata=metadata) + batch = pa.record_batch( + [pa.array([Decimal("12345678901234567890.1234567890"), None], pa.decimal128(38, 10))], + schema=pa.schema([field]), + ) + assert stream_reader(stream_bytes(batch.schema)).schema == batch.schema + crossed = one_batch(batch_bytes(batch)) + assert crossed.equals(batch) + assert crossed.schema.equals(batch.schema, check_metadata=True) + reader = stream_reader(batch_bytes(batch)) + assert isinstance(reader, pa.RecordBatchReader) # a stream: batches are read on demand + assert reader.read_next_batch().equals(batch) + + +def test_a_dictionary_column_crosses_in_its_own_stream() -> None: + batch = pa.record_batch({"zone": pa.array(["north", "south", "north"]).dictionary_encode()}) + assert one_batch(batch_bytes(batch)).equals(batch) diff --git a/packages/server/tests/store_faults.py b/packages/server/tests/store_faults.py new file mode 100644 index 00000000..8859093e --- /dev/null +++ b/packages/server/tests/store_faults.py @@ -0,0 +1,66 @@ +"""Faults the server's tests inject at the store contract's binding: one operation's call wrapped, +so a test acts around it (a crash, an interrupt, another writer) without reaching into the +store's internals (#486 G2).""" + +from __future__ import annotations + +import dataclasses +import uuid +from collections.abc import Callable +from typing import Any + +from orvex.contracts.binding import Bound +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.v1 import StatementRequest +from orvex.contracts.store.values import LakeHandle + +STORE = "store" + + +@dataclasses.dataclass(frozen=True, slots=True) +class Call: + """One intercepted call: the operation sent on, and the store to send others to.""" + + bound: Bound + envelope: CallEnvelope + arguments: tuple[Any, ...] + proceed: Callable[[], Any] + + def send(self, operation: str, *arguments: object) -> Any: # noqa: ANN401 - its result + key = uuid.uuid4().hex if self.bound.contract.operation(operation).changes_state else None + envelope = dataclasses.replace(self.envelope, idempotency_key=key) + return self.bound.call(operation, envelope, *arguments) + + def another_writer(self, sql: str) -> None: + """Another writer of the one open lake commits `sql` in a transaction of its own.""" + (lake,) = self.send("held").lakes + transaction = self.send("begin", LakeHandle(lake)) + self.send( + "execute", transaction, StatementRequest(kind="statement", sql=sql, parameters=[]) + ) + self.send("commit", transaction) + + +def conflict() -> StoreError: + """The store's refusal of a commit another writer got to first.""" + return StoreError( + "STORE_TRANSACTION_CONFLICT", + "Another writer changed the same data first; nothing was written.", + {}, + ) + + +def around(operation: str, wrapper: Callable[[Call], Any]) -> Callable[[], None]: + """Every store call of `operation` goes through `wrapper`; the returned function undoes it.""" + real = Bound.call + + def call(self: Bound, name: str, envelope: object, /, *arguments: Any, **keywords: Any) -> Any: # noqa: ANN401 + if self.contract.name != STORE or name != operation: + return real(self, name, envelope, *arguments, **keywords) + assert isinstance(envelope, CallEnvelope) + proceed = lambda: real(self, name, envelope, *arguments, **keywords) # noqa: E731 + return wrapper(Call(self, envelope, arguments, proceed)) + + Bound.call = call # type: ignore[method-assign] + return lambda: setattr(Bound, "call", real) diff --git a/packages/server/tests/test_clickhouse_wiring.py b/packages/server/tests/test_clickhouse_wiring.py index e892e0ec..a84dbe53 100644 --- a/packages/server/tests/test_clickhouse_wiring.py +++ b/packages/server/tests/test_clickhouse_wiring.py @@ -11,6 +11,7 @@ import pytest from orvex.contracts.project.v1 import ClickhouseConnection +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import TablesResult from orvex.semantic.evidence import EvidenceRecord from orvex.server import cli, clickhouse_engines, development_switch @@ -20,7 +21,6 @@ from orvex.server.identity import load_identity from orvex.server.project import load_project from orvex.server.semantic_model import ExternalTables, Loaded, load_semantic -from orvex.store.errors import StoreError from .test_lake_separation import make_project from .test_semantic_model import VENDORS, codes, extension, model, opened, ran diff --git a/packages/server/tests/test_commands.py b/packages/server/tests/test_commands.py index 84c39e68..6ac663f4 100644 --- a/packages/server/tests/test_commands.py +++ b/packages/server/tests/test_commands.py @@ -13,13 +13,13 @@ import duckdb import pytest +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.server import commands from orvex.server import run as run_command from orvex.server.cli import main from orvex.server.commands import project_lock from orvex.server.identity import load_identity -from orvex.store.errors import StoreError -from orvex.store.lake import Lake REPOSITORY = Path(__file__).resolve().parents[3] SLUG = load_identity().slug diff --git a/packages/server/tests/test_evidence_runs.py b/packages/server/tests/test_evidence_runs.py index bd47c202..e6e246d8 100644 --- a/packages/server/tests/test_evidence_runs.py +++ b/packages/server/tests/test_evidence_runs.py @@ -11,12 +11,12 @@ from typing import Any import duckdb +from orvex.contracts.store.client import Lake from orvex.semantic.evidence import Ledger from orvex.server import commands, evidence_runs from orvex.server.identity import load_identity from orvex.server.project import load_project from orvex.server.semantic_model import load_semantic -from orvex.store.lake import Lake IDENTITY = load_identity() VENDORS = (IDENTITY.ossie_vendor_name, *IDENTITY.previous_vendor_names) diff --git a/packages/server/tests/test_existing_lakes.py b/packages/server/tests/test_existing_lakes.py index a39de84a..0489e420 100644 --- a/packages/server/tests/test_existing_lakes.py +++ b/packages/server/tests/test_existing_lakes.py @@ -19,14 +19,14 @@ import duckdb import pytest from orvex.access.local import LocalRequest +from orvex.contracts.store.errors import StoreError from orvex.server import cli, existing_lakes, semantic_http from orvex.server.commands import project_lock from orvex.server.existing_lakes import ExistingLakes, memory_share from orvex.server.identity import load_identity from orvex.server.instance import ProjectRegistry, home_variable from orvex.server.project import load_project -from orvex.store.errors import StoreError -from orvex.store.existing_lake import ExistingLake +from orvex.server.store_access import store from .test_lake_separation import other_device # noqa: F401 - a fixture from .test_semantic_model import extension, model, ran @@ -154,13 +154,13 @@ def test_the_managed_lake_and_each_existing_lake_share_the_budget( monkeypatch.setattr(cli, "memory_limit", lambda: "900MB") monkeypatch.setattr(cli, "instance_directory", lambda identity: served.instance) given: list[str] = [] - real = ExistingLake + real = existing_lakes.open_existing_lake - def recorded(attachment: Any, *, memory_limit: str, threads: int) -> Any: # noqa: ANN401 - given.append(memory_limit) - return real(attachment, memory_limit=memory_limit, threads=threads) + def recorded(client: Any, attachment: Any, limits: Any) -> Any: # noqa: ANN401 + given.append(limits.memory_limit) + return real(client, attachment, limits) - monkeypatch.setattr(existing_lakes, "ExistingLake", recorded) + monkeypatch.setattr(existing_lakes, "open_existing_lake", recorded) opened_with: list[str | None] = [] monkeypatch.setattr( cli, "open_existing", lambda home, place, memory=None: opened_with.append(memory) @@ -324,21 +324,14 @@ def test_a_check_that_cannot_have_the_project_holds_a_new_snapshot_back(served: def test_a_lake_that_cannot_be_attached_leaves_none_open( served: Served, monkeypatch: pytest.MonkeyPatch ) -> None: - closed: list[str] = [] - real = ExistingLake.close - - def close(self: ExistingLake) -> None: - closed.append(self.name) - real(self) - - monkeypatch.setattr(ExistingLake, "close", close) + held = store().call("held").existing_lakes text = (served.folder / "connections.yaml").read_text() broken = Path(tempfile.mkdtemp(dir="/dev/shm")) / "missing.sqlite" second = connection(Owner(broken, served.owner.data)).replace(" billing:", " zz_broken:") (served.folder / "connections.yaml").write_text(text + second) with pytest.raises(StoreError), opened(served): pass - assert closed == ["billing"] + assert store().call("held").existing_lakes == held # nothing attached stays open shutil.rmtree(broken.parent, ignore_errors=True) diff --git a/packages/server/tests/test_lake_catalog.py b/packages/server/tests/test_lake_catalog.py index 81502138..7a21af78 100644 --- a/packages/server/tests/test_lake_catalog.py +++ b/packages/server/tests/test_lake_catalog.py @@ -15,13 +15,14 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ReadRequest +from orvex.contracts.store.values import Limits from orvex.server import commands, lake_open, lake_place, mounts from orvex.server.cli import main from orvex.server.identity import load_identity from orvex.server.project import ProjectLoadError, load_project -from orvex.store.errors import StoreError -from orvex.store.lake import Limits, open_lake +from orvex.server.store_access import store SLUG = load_identity().slug PASSWORD_ENV = "STORE_TEST_POSTGRES_PASSWORD" @@ -72,7 +73,7 @@ def home(folder: Path) -> Path: def count(folder: Path) -> int: """The model's rows, read through the project's own lake location.""" place = commands.lake_location(load_project(folder), home(folder)) - lake = open_lake(place.location, LIMITS) + lake = store().open_lake(place.location, LIMITS) try: request = ReadRequest( kind="read", diff --git a/packages/server/tests/test_lake_data.py b/packages/server/tests/test_lake_data.py index 240a4554..c2a3d52f 100644 --- a/packages/server/tests/test_lake_data.py +++ b/packages/server/tests/test_lake_data.py @@ -19,11 +19,12 @@ import pytest from moto.server import ThreadedMotoServer from orvex.contracts.store.v1 import ReadRequest +from orvex.contracts.store.values import Limits from orvex.server import commands, lake_place from orvex.server.cli import main from orvex.server.identity import load_identity from orvex.server.project import load_project -from orvex.store.lake import Limits, open_lake +from orvex.server.store_access import store from orvex.testing import azurite @@ -89,7 +90,7 @@ def problems(folder: Path) -> list[str]: def count(folder: Path) -> int: place = commands.lake_location(load_project(folder), home(folder)) - lake = open_lake(place.location, LIMITS) + lake = store().open_lake(place.location, LIMITS) try: request = ReadRequest( kind="read", diff --git a/packages/server/tests/test_lake_refusals.py b/packages/server/tests/test_lake_refusals.py index ec70182f..a64ba52d 100644 --- a/packages/server/tests/test_lake_refusals.py +++ b/packages/server/tests/test_lake_refusals.py @@ -2,8 +2,8 @@ from __future__ import annotations +from orvex.contracts.store.errors import StoreError from orvex.server.lake_refusals import as_semantic -from orvex.store.errors import StoreError def test_a_lakes_data_refused_keeps_the_connection_and_the_reason_only() -> None: diff --git a/packages/server/tests/test_lake_separation.py b/packages/server/tests/test_lake_separation.py index 27efefdd..3ad7cb72 100644 --- a/packages/server/tests/test_lake_separation.py +++ b/packages/server/tests/test_lake_separation.py @@ -10,12 +10,12 @@ from pathlib import Path import pytest +from orvex.contracts.store.errors import StoreError from orvex.server import lake_separation from orvex.server.commands import check_project from orvex.server.identity import load_identity from orvex.server.instance import ProjectRegistry from orvex.server.problems import Problem -from orvex.store.errors import StoreError SLUG = load_identity().slug PROJECT = "version: 1\nname: {name}\ntime_zone: UTC\n" diff --git a/packages/server/tests/test_maintenance.py b/packages/server/tests/test_maintenance.py index be9f7e78..9b4e907a 100644 --- a/packages/server/tests/test_maintenance.py +++ b/packages/server/tests/test_maintenance.py @@ -13,12 +13,12 @@ import duckdb import pytest from orvex.contracts.maintenance_record.v1 import MaintenanceRecord +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.server import commands, maintenance, run_metadata from orvex.server.cli import main from orvex.server.identity import load_identity from orvex.server.scheduler import schedule -from orvex.store.errors import StoreError -from orvex.store.lake import Lake IDENTITY = load_identity() SLUG = IDENTITY.slug diff --git a/packages/server/tests/test_permission_state.py b/packages/server/tests/test_permission_state.py index e37b395f..f7c64740 100644 --- a/packages/server/tests/test_permission_state.py +++ b/packages/server/tests/test_permission_state.py @@ -8,10 +8,10 @@ from typing import Any import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ChangesRequest, ChangesResult from orvex.semantic.errors import SemanticError from orvex.server.permission_state import PermissionState, permission_state -from orvex.store.errors import StoreError OLD = "sha256:" + "a" * 64 NEW = "sha256:" + "b" * 64 diff --git a/packages/server/tests/test_query_cancel.py b/packages/server/tests/test_query_cancel.py index ab153536..47327b2d 100644 --- a/packages/server/tests/test_query_cancel.py +++ b/packages/server/tests/test_query_cancel.py @@ -8,8 +8,8 @@ from typing import Any import pytest +from orvex.contracts.store.errors import StoreError from orvex.server.query_cancel import Cancel -from orvex.store.errors import StoreError class Handle: diff --git a/packages/server/tests/test_run_records.py b/packages/server/tests/test_run_records.py index ef99f484..9b6979ea 100644 --- a/packages/server/tests/test_run_records.py +++ b/packages/server/tests/test_run_records.py @@ -14,20 +14,21 @@ import zoneinfo from collections.abc import Iterator from pathlib import Path +from types import SimpleNamespace from typing import Any import duckdb -import orvex.store.transaction as store_transaction import pytest from orvex.contracts.run_record.v1 import RunRecord +from orvex.contracts.store.client import Lake, Transaction +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import LocalLakeLocation, ReadRequest, StatementRequest from orvex.server import commands, lake_open, maintenance from orvex.server import run as run_command from orvex.server.cli import main from orvex.server.identity import load_identity from orvex.server.project import load_project -from orvex.store.errors import StoreError -from orvex.store.lake import Lake, Transaction, open_lake +from orvex.server.store_access import store from orvex.transform.run_records import ( history_request, last_run_request, @@ -35,6 +36,8 @@ record_statements, ) +from . import store_faults + SLUG = load_identity().slug PROJECT = "version: 1\nname: Test\ntime_zone: America/La_Paz\n" CONNECTIONS = "version: 1\nconnections:\n archive:\n type: parquet\n path: data/r.parquet\n" @@ -118,14 +121,6 @@ def records(folder: Path) -> list[dict[str, object]]: return rows -def snapshot_of_token(folder: Path, token: str) -> int: - ((snapshot,),) = lake_rows( - folder, - f"SELECT snapshot_id FROM ducklake_snapshots('lake') WHERE commit_extra_info = '{token}'", - ) - return int(str(snapshot)) - - def latest_snapshot(folder: Path) -> int: ((snapshot,),) = lake_rows(folder, "SELECT max(snapshot_id) FROM ducklake_snapshots('lake')") return int(str(snapshot)) @@ -218,7 +213,7 @@ def recorded_lake(folder: Path, record: RunRecord) -> Iterator[Lake]: """A lake of its own, opened through the store's public API, holding `record`: the store binds every parameter of a statement or refuses it (#246 item 6).""" location = LocalLakeLocation(kind="lake_location", catalog="local", path=str(folder)) - lake = open_lake(location, lake_open._limits()) + lake = store().open_lake(location, lake_open._limits()) try: with lake.transaction() as transaction: for statement in record_statements([record]): @@ -472,7 +467,7 @@ def test_run_failures_no_model_caused_are_recorded_with_their_reason( ) -> None: project(tmp_path, MODELS) lake = FakeLake(fail) - monkeypatch.setattr(lake_open, "open_lake", lambda *args, **kwargs: lake) + monkeypatch.setattr(lake_open, "store", lambda: SimpleNamespace(open_lake=lambda *a, **k: lake)) exit_code, out = command(capsys, "run", str(tmp_path)) assert exit_code == 1 assert code in out @@ -495,17 +490,20 @@ def test_the_record_names_the_run_s_own_snapshot_when_another_writer_commits_rig # #255: two clones sharing one lake; the other commits right after this run's commit. The # records name the run's snapshot, and rows are counted on it. project(tmp_path, MODELS) - real = store_transaction._commit injected: list[int] = [] - def commit_then_another_writer(cursor: duckdb.DuckDBPyConnection) -> None: - real(cursor) + def commit_then_another_writer(call: store_faults.Call) -> object: + committed = call.proceed() if not injected: injected.append(1) - cursor.execute("CREATE TABLE lake.main.other_clone AS SELECT 1 AS x") # own commit + call.another_writer("CREATE TABLE main.other_clone AS SELECT 1 AS x") + return committed - monkeypatch.setattr(store_transaction, "_commit", commit_then_another_writer) - code, out = command(capsys, "run", str(tmp_path)) + undo = store_faults.around("commit", commit_then_another_writer) + try: + code, out = command(capsys, "run", str(tmp_path)) + finally: + undo() assert code == 0, out recorded = printed_snapshot(out) rows = [r for r in records(tmp_path) if r["outcome"] == "success"] @@ -526,8 +524,8 @@ def commit_then_another_writer(cursor: duckdb.DuckDBPyConnection) -> None: CHILD = textwrap.dedent( """ import os, sys - import orvex.store.lake as store_lake - import orvex.store.transaction as store_transaction + sys.path.insert(0, sys.argv[4]) + import store_faults from orvex.server import run as run_command from orvex.server.cli import main @@ -539,29 +537,23 @@ def arming(items): armed.append(True) return records(items) - committed = store_lake.Lake._committed - - def killed(self, token): + def killed(call): + committed = call.proceed() if armed: # the run's own commit just happened: the process ends here with open(token_file, "w", encoding="utf-8") as handle: - handle.write(token) + handle.write(str(committed.snapshot.id)) os._exit(3) - return committed(self, token) + return committed - commit = store_transaction._commit - - def another_writer_first(cursor): + def another_writer_first(call): if len(armed) == 1: # the run's block ran: another writer commits the same table - other = cursor.cursor() - other.execute("DROP TABLE lake.s.a") - other.close() - commit(cursor) + call.send("rollback", *call.arguments) + call.another_writer("DROP TABLE s.a") + raise store_faults.conflict() + return call.proceed() run_command.record_statements = arming - if mode == "kill": - store_lake.Lake._committed = killed - else: - store_transaction._commit = another_writer_first + store_faults.around("commit", killed if mode == "kill" else another_writer_first) code = main(["run", folder]) with open(token_file, "w", encoding="utf-8") as handle: handle.write(f"{code} {len(armed)}") @@ -573,7 +565,7 @@ def child_run(folder: Path, mode: str, result: Path) -> subprocess.CompletedProc """`run` in a process of its own, stopped or disturbed as `mode` says. A process of its own: DuckDB 2.0.0 keeps a local catalog attached in the process after a commit conflict.""" return subprocess.run( - [sys.executable, "-c", CHILD, str(folder), mode, str(result)], + [sys.executable, "-c", CHILD, str(folder), mode, str(result), str(Path(__file__).parent)], capture_output=True, text=True, env=os.environ.copy(), @@ -587,7 +579,7 @@ def test_a_kill_right_after_the_commit_leaves_data_and_records(tmp_path: Path) - token_file = tmp_path / "token" child = child_run(folder, "kill", token_file) assert child.returncode == 3, child.stderr - committed = snapshot_of_token(folder, token_file.read_text(encoding="utf-8")) + committed = int(token_file.read_text(encoding="utf-8")) assert lake_rows(folder, "SELECT count(*) FROM lake.s.a") == [(5,)] rows = records(folder) assert [(r["model"], r["outcome"]) for r in rows] == [ @@ -604,18 +596,21 @@ def test_ctrl_c_right_after_the_commit_adds_no_failed_record( # Criterion 1b: the run is committed and recorded; the interruption still goes on. project(tmp_path, MODELS) armed = armed_on_records(monkeypatch) - tokens: list[str] = [] - real = Lake._committed + snapshots: list[int] = [] - def interrupted(self: Lake, token: str) -> object: + def interrupted(call: store_faults.Call) -> object: + committed = call.proceed() if armed: - tokens.append(token) + snapshots.append(committed.snapshot.id) raise KeyboardInterrupt - return real(self, token) + return committed - monkeypatch.setattr(Lake, "_committed", interrupted) - with pytest.raises(KeyboardInterrupt): - main(["run", str(tmp_path)]) + undo = store_faults.around("commit", interrupted) + try: + with pytest.raises(KeyboardInterrupt): + main(["run", str(tmp_path)]) + finally: + undo() out = capsys.readouterr().out run_id = re.search(r"^run (\S+)$", out, re.MULTILINE) assert run_id @@ -626,7 +621,7 @@ def interrupted(self: Lake, token: str) -> object: (run_id.group(1), "s.b", "success"), (run_id.group(1), "s.c", "success"), ] - assert {r["snapshot_id"] for r in rows} == {snapshot_of_token(tmp_path, tokens[0])} + assert {r["snapshot_id"] for r in rows} == {snapshots[0]} def test_ctrl_c_before_the_commit_commits_nothing_and_records_the_run_failed( @@ -635,16 +630,20 @@ def test_ctrl_c_before_the_commit_commits_nothing_and_records_the_run_failed( # Criterion 2. project(tmp_path, MODELS) armed = armed_on_records(monkeypatch) - real = store_transaction._commit - def interrupted(cursor: duckdb.DuckDBPyConnection) -> None: + def interrupted(call: store_faults.Call) -> object: if armed and len(armed) == 1: + # As the store does on an interrupt inside its commit: nothing is committed. + call.send("rollback", *call.arguments) raise KeyboardInterrupt - real(cursor) + return call.proceed() - monkeypatch.setattr(store_transaction, "_commit", interrupted) - with pytest.raises(KeyboardInterrupt): - main(["run", str(tmp_path)]) + undo = store_faults.around("commit", interrupted) + try: + with pytest.raises(KeyboardInterrupt): + main(["run", str(tmp_path)]) + finally: + undo() capsys.readouterr() assert [(r["outcome"], r["error_code"], r["snapshot_id"]) for r in records(tmp_path)] == [ ("failed", "STORE_CANCELLED", None) diff --git a/packages/server/tests/test_schedule.py b/packages/server/tests/test_schedule.py index 29eba967..e23e926d 100644 --- a/packages/server/tests/test_schedule.py +++ b/packages/server/tests/test_schedule.py @@ -16,12 +16,12 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.server import commands, lake_open, lake_place, project_check, run_metadata, scheduler from orvex.server import run as run_command from orvex.server.cli import main from orvex.server.identity import Identity, load_identity from orvex.server.scheduler import schedule -from orvex.store.errors import StoreError SLUG = load_identity().slug PROJECT = "version: 1\nname: Test\ntime_zone: UTC\n" diff --git a/packages/server/tests/test_semantic_explain.py b/packages/server/tests/test_semantic_explain.py index 29cda268..bafc5f62 100644 --- a/packages/server/tests/test_semantic_explain.py +++ b/packages/server/tests/test_semantic_explain.py @@ -113,7 +113,7 @@ def test_a_builder_sees_the_estimate_and_the_materialization( def test_an_explanation_never_reads( served: tuple[LocalSemantic, TestClient], monkeypatch: pytest.MonkeyPatch ) -> None: - from orvex.store.lake import Lake + from orvex.contracts.store.client import Lake semantic, http = served original = Lake.read diff --git a/packages/server/tests/test_semantic_http.py b/packages/server/tests/test_semantic_http.py index 0d25d221..0b0b8134 100644 --- a/packages/server/tests/test_semantic_http.py +++ b/packages/server/tests/test_semantic_http.py @@ -21,7 +21,7 @@ from orvex.server.app import LocalSemantic, create_app from orvex.server.local_access import LocalAccess, new_access, serving from orvex.server.project import load_project -from orvex.store.json_encoding import arrow_to_json +from orvex.server.store_access import store from tests.support.semantic_http import ( BASE, @@ -390,7 +390,7 @@ def assert_equivalent(as_json: dict[str, Any], table: pa.Table) -> None: assert metadata[semantic_http.RECEIPT_KEY] == canonicalize(as_json["receipt"]) assert json.loads(metadata[b"semantic.cell_states"]) == result.get("cell_states", []) text = "".join( - arrow_to_json(pa.RecordBatchReader.from_batches(table.schema, table.to_batches())) + store().arrow_to_json(pa.RecordBatchReader.from_batches(table.schema, table.to_batches())) ) values = json.loads(text)["rows"] states = {(s["row"], s["column"]) for s in result.get("cell_states", [])} @@ -577,7 +577,7 @@ async def send(message: MutableMapping[str, Any]) -> None: def test_a_store_failure_is_a_500_with_a_request_id_only() -> None: - from orvex.store.errors import StoreError + from orvex.contracts.store.errors import StoreError class Failing: def session(self) -> Any: # noqa: ANN401 @@ -610,7 +610,7 @@ def test_a_lakes_data_refused_by_the_store_is_a_503_with_its_code_and_nothing_el """#447 (SEC-DQ-010, -011, -014, -015): an existing lake whose data does not pass its checks reaches the caller as `STORE_SOURCE_INVALID` with the connection, never as an internal error, and nothing of the store's message or details (files, DuckDB) comes with it.""" - from orvex.store.errors import StoreError + from orvex.contracts.store.errors import StoreError class Refused: def session(self) -> Any: # noqa: ANN401 @@ -669,7 +669,7 @@ def test_cell_states_and_the_others_row_are_the_same_in_arrow_and_json() -> None states = [CellState(2, 0, "others"), CellState(2, 2, "undefined_at_total")] result = built(data, columns, states=states, others_row=2) receipt = {"kind": "receipt"} - document = {"result": json.loads(result.wire(arrow_to_json)), "receipt": receipt} + document = {"result": json.loads(result.wire(store().arrow_to_json)), "receipt": receipt} sink = pa.BufferOutputStream() answer = semantic_http.arrow_answer(result, receipt) with pa.ipc.new_stream(sink, answer.schema) as writer: diff --git a/packages/server/tests/test_semantic_model.py b/packages/server/tests/test_semantic_model.py index 04c82aba..9730b48a 100644 --- a/packages/server/tests/test_semantic_model.py +++ b/packages/server/tests/test_semantic_model.py @@ -13,11 +13,11 @@ import duckdb import pytest +from orvex.contracts.store.client import Lake from orvex.server import app, commands, semantic_model from orvex.server.identity import load_identity from orvex.server.project import load_project from orvex.server.semantic_model import Loaded, SemanticModel, load_semantic -from orvex.store.lake import Lake IDENTITY = load_identity() VENDORS = (IDENTITY.ossie_vendor_name, *IDENTITY.previous_vendor_names) diff --git a/packages/server/tests/test_since.py b/packages/server/tests/test_since.py index 8bcb9ff5..985815c1 100644 --- a/packages/server/tests/test_since.py +++ b/packages/server/tests/test_since.py @@ -6,12 +6,12 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import StatementRequest from orvex.server import run as run_command from orvex.server import run_metadata from orvex.server.cli import main from orvex.server.identity import load_identity -from orvex.store.errors import StoreError SLUG = load_identity().slug PROJECT = "version: 1\nname: Test\ntime_zone: America/La_Paz\n" diff --git a/packages/store/benchmarks/source_stream.py b/packages/store/benchmarks/source_stream.py new file mode 100644 index 00000000..83fde5a5 --- /dev/null +++ b/packages/store/benchmarks/source_stream.py @@ -0,0 +1,169 @@ +"""Source isolation cost (#372 §5), the store's own benchmark of its internals: landing one +database-source table three ways. + + python packages/store/benchmarks/source_stream.py --rows 10000000 --rows 50000000 + +- `direct`: the source attached to the reading DuckDB instance and copied with + `CREATE TABLE … AS`, the path a lake used before #372 (no isolation); +- `stream`: a second DuckDB instance in the same process, its result streamed as Arrow + batches into the reading instance (in-process isolation, rejected by ADR-0031 §6); +- `worker`: the product's path, `source_worker.start`, an Arrow IPC stream from a worker + process (ADR-0031). + +Each path runs `--runs` times per size (default 3) on the same synthetic table (BIGINT, +DECIMAL(18,2), TIMESTAMPTZ, VARCHAR); the report gives the median wall time, the rows per +second and the peak resident memory of this (the reading) process. Every run checks the +landed row count and the exact sum of the decimal column against the source's. Nothing is +estimated: a failed check fails the benchmark. +""" + +from __future__ import annotations + +import argparse +import json +import platform +import resource +import statistics +import sys +import tempfile +import time +from collections.abc import Callable +from pathlib import Path + +import duckdb +from orvex.contracts.store.v1 import FileSourceAttachment +from orvex.store import source_worker + +MEMORY = "4GB" +THREADS = 4 +BATCH = 65_536 + + +def build(path: Path, rows: int) -> tuple[int, str]: + """The synthetic source table, and its row count and exact sum, computed by DuckDB.""" + with duckdb.connect(str(path)) as writer: + writer.sql( + "CREATE TABLE readings AS SELECT range AS id, " + "CAST((range % 100000) / 100 AS DECIMAL(18,2)) AS amount, " + "TIMESTAMPTZ '2026-01-01 00:00:00+00' + to_seconds(range % 31536000) AS at, " + f"'meter-' || CAST(range % 1000 AS VARCHAR) AS meter FROM range({int(rows)})" + ) + row = writer.execute( + "SELECT count(*), CAST(sum(amount) AS VARCHAR) FROM readings" + ).fetchone() + assert row is not None + return int(row[0]), str(row[1]) + + +def target() -> duckdb.DuckDBPyConnection: + return duckdb.connect(config={"memory_limit": MEMORY, "threads": THREADS}) + + +def check(connection: duckdb.DuckDBPyConnection, expected: tuple[int, str]) -> None: + row = connection.execute("SELECT count(*), CAST(sum(amount) AS VARCHAR) FROM landed").fetchone() + if row is None or (int(row[0]), str(row[1])) != expected: + raise SystemExit(f"wrong landing: {row} != {expected}") + + +def direct(path: Path, expected: tuple[int, str]) -> None: + with target() as connection: + connection.sql(f"ATTACH '{path}' AS source (READ_ONLY)") + connection.sql("CREATE TABLE landed AS SELECT * FROM source.readings") + check(connection, expected) + + +def stream(path: Path, expected: tuple[int, str]) -> None: + with duckdb.connect(config={"memory_limit": MEMORY, "threads": THREADS}) as source: + source.sql("SET arrow_lossless_conversion = true") + source.sql(f"ATTACH '{path}' AS source (READ_ONLY)") + reader = source.sql("SELECT * FROM source.readings").to_arrow_reader(BATCH) + with target() as connection: + connection.register("incoming", reader) + connection.sql("CREATE TABLE landed AS SELECT * FROM incoming") + check(connection, expected) + + +def worker(path: Path, expected: tuple[int, str]) -> None: + attachment = FileSourceAttachment( + kind="source_attachment", name="source", type="duckdb", path=str(path) + ) + started = source_worker.start( + attachment, + "", + "SELECT * FROM source.readings", + [], + memory_limit=MEMORY, + threads=THREADS, + timeout_ms=3_600_000, + batch_size=BATCH, + folder=path.parent, + ).open() + try: + with target() as connection: + connection.register("incoming", started.reader()) + connection.sql("CREATE TABLE landed AS SELECT * FROM incoming") + check(connection, expected) + finally: + started.close() + + +PATHS: dict[str, Callable[[Path, tuple[int, str]], None]] = { + "direct": direct, + "stream": stream, + "worker": worker, +} + + +def peak_mib() -> float: + """Peak resident memory of this process (the reading side), in MiB. A worker's own peak is + not reported: a spawned child's `ru_maxrss` counts the parent's pages from before `exec`.""" + scale = 1024 * 1024 if sys.platform == "darwin" else 1024 + return round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * scale / 2**20, 1) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--rows", type=int, action="append", required=True) + parser.add_argument("--runs", type=int, default=3) + arguments = parser.parse_args() + report: dict[str, object] = { + "python": platform.python_version(), + "duckdb": duckdb.__version__, + "machine": platform.platform(), + "cpus": len(__import__("os").sched_getaffinity(0)), + "memory_limit": MEMORY, + "threads": THREADS, + "batch_size": BATCH, + "results": [], + } + results: list[dict[str, object]] = [] + for rows in arguments.rows: + with tempfile.TemporaryDirectory(prefix="source-stream-") as folder: + path = Path(folder) / "source.duckdb" + expected = build(path, rows) + for name, land in PATHS.items(): + times = [] + for _ in range(arguments.runs): + started = time.perf_counter() + land(path, expected) + times.append(time.perf_counter() - started) + median = statistics.median(times) + own = peak_mib() + results.append( + { + "rows": rows, + "path": name, + "seconds": [round(t, 3) for t in times], + "median_s": round(median, 3), + "rows_per_s": round(rows / median), + "peak_rss_mib": own, + } + ) + print(json.dumps(results[-1]), flush=True) + report["results"] = results + print(json.dumps(report, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/store/tests/contract_client.py b/packages/store/tests/contract_client.py new file mode 100644 index 00000000..dba0f13d --- /dev/null +++ b/packages/store/tests/contract_client.py @@ -0,0 +1,16 @@ +"""The store's own tests reaching its implementation as a consumer does: through the store +contract's in-process binding and its client (`orvex.contracts.store.client`).""" + +from __future__ import annotations + +from orvex.contracts.binding import Bound +from orvex.contracts.store.client import Caller, StoreClient +from orvex.contracts.store.interface import STORE +from orvex.store.api import Store + +CALLER = Caller(actor="store-tests", project_id="store-tests", revision="1", correlation_id="t") + + +def store_client() -> StoreClient: + """A client of a fresh `Store`.""" + return StoreClient(Bound(STORE, Store(), serialized=False), CALLER) diff --git a/packages/store/tests/fixtures/lake_interface.json b/packages/store/tests/fixtures/lake_interface.json index 84bf405d..fb632985 100644 --- a/packages/store/tests/fixtures/lake_interface.json +++ b/packages/store/tests/fixtures/lake_interface.json @@ -11,7 +11,6 @@ "_require_limits", "_single_statement", "_snapshot_of", - "_source_not_found", "_source_password", "_storage_credentials", "_typed" diff --git a/tests/support/parquet_footer.py b/packages/store/tests/parquet_footer.py similarity index 100% rename from tests/support/parquet_footer.py rename to packages/store/tests/parquet_footer.py diff --git a/packages/store/tests/test_admission_edges.py b/packages/store/tests/test_admission_edges.py index 07d738ab..40094857 100644 --- a/packages/store/tests/test_admission_edges.py +++ b/packages/store/tests/test_admission_edges.py @@ -14,6 +14,7 @@ from types import SimpleNamespace import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresLakeLocation, @@ -22,7 +23,6 @@ ) from orvex.store import lake as lake_module from orvex.store import read_stream -from orvex.store.errors import StoreError from orvex.store.lake import Guard, Lake, Limits, ReadHandle, open_lake ROUNDS = 50 diff --git a/packages/store/tests/test_budgets.py b/packages/store/tests/test_budgets.py index 6f7e2e00..32b2dfbb 100644 --- a/packages/store/tests/test_budgets.py +++ b/packages/store/tests/test_budgets.py @@ -8,6 +8,7 @@ from collections.abc import Iterator import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresLakeLocation, @@ -16,7 +17,6 @@ ) from orvex.store import lake as lake_module from orvex.store import read_stream -from orvex.store.errors import StoreError from orvex.store.lake import Admission, Lake, Limits, open_lake SLOW = "SELECT sum(hash(a.range + b.range)) AS n FROM range(200000000) AS a, range(1000) AS b" diff --git a/packages/store/tests/test_catalog_limits.py b/packages/store/tests/test_catalog_limits.py index c357f4a4..bbf386fc 100644 --- a/packages/store/tests/test_catalog_limits.py +++ b/packages/store/tests/test_catalog_limits.py @@ -11,6 +11,7 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresLakeLocation, @@ -19,7 +20,6 @@ ) from orvex.store import read_tree from orvex.store import transaction as transaction_module -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, Transaction, open_lake type Location = LocalLakeLocation | PostgresLakeLocation diff --git a/packages/store/tests/test_catalog_location.py b/packages/store/tests/test_catalog_location.py index 50f97e3d..a7710df6 100644 --- a/packages/store/tests/test_catalog_location.py +++ b/packages/store/tests/test_catalog_location.py @@ -15,6 +15,7 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresLakeLocation, @@ -22,7 +23,6 @@ SqliteLakeLocation, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.isolation import _sandbox_entries from orvex.store.lake import Lake, Limits, open_lake from orvex.store.locations import ( diff --git a/packages/store/tests/test_changes.py b/packages/store/tests/test_changes.py index b637b923..05d91178 100644 --- a/packages/store/tests/test_changes.py +++ b/packages/store/tests/test_changes.py @@ -6,8 +6,8 @@ from pathlib import Path import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ChangesRequest, LakeLocation, StatementRequest -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_clickhouse.py b/packages/store/tests/test_clickhouse.py index 363402a5..f11215d6 100644 --- a/packages/store/tests/test_clickhouse.py +++ b/packages/store/tests/test_clickhouse.py @@ -18,10 +18,10 @@ import pyarrow as pa import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter from orvex.store import clickhouse from orvex.store.clickhouse import Budget, ClickHouse, ClickHouseTarget, duckdb_type -from orvex.store.errors import StoreError BUDGET = Budget(timeout_ms=30_000, memory_bytes=2_000_000_000) SAFE = { diff --git a/packages/store/tests/test_clickhouse_scanner_fuzzing.py b/packages/store/tests/test_clickhouse_scanner_fuzzing.py new file mode 100644 index 00000000..383c7d9f --- /dev/null +++ b/packages/store/tests/test_clickhouse_scanner_fuzzing.py @@ -0,0 +1,172 @@ +"""P3-14 (#448; #417): the ClickHouse statement scanner (`orvex.store.clickhouse._bound`), +fuzzed. On statements built from the adapter's own pieces and the lexer's hard cases (strings +and their escapes, heredocs, quoted names, every comment form, placeholders, `;`, `FORMAT`, +`SETTINGS`), it accepts or refuses with `STORE_STATEMENT_INVALID` or `STORE_PARAMETER_INVALID`. +Every statement it accepts is checked against `clickhouse_view`, a lexer written here from +ClickHouse's documented syntax, independently of the scanner: what ClickHouse would run is one +`SELECT` (or `WITH`), with no `;` and no `FORMAT`, `SETTINGS` or `INTO OUTFILE`. A scanner that +agrees only with itself proves nothing. `PROPERTY_PROFILE=nightly` explores further. +""" + +from __future__ import annotations + +import os +import re + +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.v1 import Parameter +from orvex.store import clickhouse + +NIGHTLY = os.environ.get("PROPERTY_PROFILE") == "nightly" +PROFILE = settings( + max_examples=400 if NIGHTLY else 10, + derandomize=not NIGHTLY, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +# --- the ClickHouse scanner ------------------------------------------------------------------ + +PIECES = ( + "SELECT", "WITH x AS (SELECT 1)", "(SELECT", ")", "`t`.`amount`", '"t"."zone"', "x", "1", + ",", "FROM `db`.`t`", "WHERE", "=", "AND", "GROUP BY 1", "ORDER BY 1", "LIMIT 10", + "{p0:String}", "{p1:Nullable(String)}", "{p2:Int64}", "'a'", "'it''s'", "'\\''", + "'\\\\'", "$$ raw ; FORMAT $$", "$tag$ x $tag$", "`we``ird`", '"a\\"b"', + "-- line ; FORMAT\n", "# hash SETTINGS a=1\n", "#! bang ;\n", "#x\n", "// slashes ;\n", + "/* c ; */", "/* /* nested */ SETTINGS x = 1 */", "/* /* */ '", "' */", "' -- '", "*/", + "/* open", "'open", "`open", "$x$ open", + ";", "FORMAT JSON", "SETTINGS max_threads = 1", "INTO OUTFILE 'f'", "format", "\n", " ", + "Settings", "formatDateTime(x, '%F')", "settingsx", "FORMAT_x", +) # fmt: skip +TYPES = {"p0": "String", "p1": "Nullable(String)", "p2": "Int64"} + + +@st.composite +def statements(draw: st.DrawFn) -> str: + start = draw(st.sampled_from(("SELECT", "WITH x AS (SELECT 1) SELECT", "", "(SELECT"))) + rest = draw(st.lists(st.sampled_from(PIECES), max_size=10)) + glue = draw(st.sampled_from((" ", "", "\n"))) + return glue.join([start, *rest]).strip() + + +def parameters(statement: str) -> list[Parameter]: + """One parameter for each placeholder name written anywhere, as the adapter would bind it.""" + named = sorted(set(re.findall(r"\{(p[0-9]):", statement))) + values = {"p0": ("VARCHAR", "a"), "p1": ("VARCHAR", None), "p2": ("BIGINT", "7")} + return [ + Parameter.model_validate({"name": n, "type": values[n][0], "value": values[n][1]}) + for n in named + ] + + +def clickhouse_view(text: str) -> str | None: + """What ClickHouse's lexer leaves as code, from its documented syntax (written here, apart + from the scanner): strings `'…'` with `\\` escapes and `''`; heredocs `$tag$…$tag$`; + names in backquotes or double quotes with `\\` escapes and doubled quotes; comments `--`, + `#!`, `# ` and `//` to the end of the line, and `/* … */`, which nest. Each string and name + becomes `''` or `x`, each comment a space. None when the lexer would refuse the text: an + unclosed quote or comment, or a `#` that starts no comment.""" + code, at = [], 0 + while at < len(text): + char, two = text[at], text[at : at + 2] + if two in {"--", "//", "#!", "# "}: + end = text.find("\n", at) + at = len(text) if end < 0 else end + code.append(" ") + elif char == "#": + return None + elif two == "/*": + depth, at = 1, at + 2 + while depth: + if at >= len(text): + return None + if text.startswith("/*", at): + depth, at = depth + 1, at + 2 + elif text.startswith("*/", at): + depth, at = depth - 1, at + 2 + else: + at += 1 + code.append(" ") + elif char == "$" and (tag := re.match(r"\$(\w*)\$", text[at:])): + close = text.find(tag.group(0), at + len(tag.group(0))) + if close < 0: + return None + at = close + len(tag.group(0)) + code.append(" '' ") + elif char in "'`\"": + at += 1 + while True: + if at >= len(text): + return None + if text[at] == "\\" or (text[at] == char and text[at + 1 : at + 2] == char): + at += 2 + elif text[at] == char: + at += 1 + break + else: + at += 1 + code.append(" '' " if char == "'" else " x ") + else: + code.append(char) + at += 1 + return "".join(code) + + +PLACEHOLDER = r"\{([A-Za-z_][A-Za-z0-9_]*):([^{}]+)\}" + + +def runs_as_one_select(code: str) -> bool: + plain = re.sub(PLACEHOLDER, " 0 ", code).strip() + return ( + ";" not in plain + and re.match(r"(?is)^\(*\s*(SELECT|WITH)\b", plain) is not None + and re.search(r"(?i)\b(SETTINGS|FORMAT|INTO\s+OUTFILE)\b", plain) is None + ) + + +@PROFILE +@example("SELECT 1 /* /* */ SETTINGS max_threads = 1 */") +@example("SELECT 1 // ; FORMAT JSON\n") +@example("SELECT 1 #SETTINGS max_threads = 1") +@example("SELECT 1 $$ ; $$ FORMAT JSON") +@example("SELECT '\\'' ; SELECT 2") +# Found by the Reviewer pass on #448: a nested comment made the scanner read code as a string. +@example("SELECT 1 /* /* */ ' */ SETTINGS max_execution_time = 0 -- '") +@given(statements()) +def test_what_the_scanner_accepts_runs_on_clickhouse_as_one_plain_select(statement: str) -> None: + try: + clickhouse._bound(statement, parameters(statement)) + except StoreError as error: + refused: str | None = error.code + else: + refused = None + if refused is not None: + assert refused in {"STORE_STATEMENT_INVALID", "STORE_PARAMETER_INVALID"} + return + code = clickhouse_view(statement) + assert code is None or runs_as_one_select(code), code + if code is not None: # each placeholder found once, with its written type, where it runs + written = [(n, t.strip()) for n, t in re.findall(PLACEHOLDER, code)] + assert clickhouse._scan(statement)[1] == written + + +def test_the_reference_lexer_sees_what_clickhouse_documents() -> None: + """The checks above are only as good as this lexer: each documented case, and each + property, fails on a statement built to break it.""" + assert clickhouse_view("SELECT 1 -- ; x\n") == "SELECT 1 \n" + assert clickhouse_view("SELECT /* a /* b */ c */ 1") == "SELECT 1" + assert clickhouse_view("SELECT 'a\\'b'") == "SELECT '' " + assert clickhouse_view("SELECT $t$ ; $t$") == "SELECT '' " + assert clickhouse_view("SELECT 'open") is None + assert clickhouse_view("SELECT 1 #x") is None + assert runs_as_one_select("SELECT 1") + for broken in ( + "SELECT 1; SELECT 2", + "SELECT 1 FORMAT JSON", + "DROP TABLE t", + "SELECT 1 settings a = 1", + ): + assert not runs_as_one_select(broken), broken diff --git a/packages/store/tests/test_contract_kit.py b/packages/store/tests/test_contract_kit.py new file mode 100644 index 00000000..a4e60e09 --- /dev/null +++ b/packages/store/tests/test_contract_kit.py @@ -0,0 +1,91 @@ +"""#486 G1, G3, G5: the store's implementation passes its contract's whole kit, in process and +serialized, and the kit catches an implementation that breaks one of its guarantees.""" + +from __future__ import annotations + +from importlib.resources import files + +import pyarrow as pa +import pytest +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.kit_run import kit_problems, run_kit, verify_kit +from orvex.contracts.manifest import parse_manifest +from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch +from orvex.contracts.store.interface import STORE +from orvex.contracts.store.kit import STORE_KIT +from orvex.contracts.store.v1 import StatementRequest +from orvex.contracts.store.values import ReadHandle, TransactionHandle +from orvex.store import api +from orvex.store.api import Store + + +def test_the_store_passes_its_whole_kit_in_process_and_serialized() -> None: + reports = verify_kit(STORE_KIT, Store) + assert {report.serialized for report in reports} == {False, True} + for report in reports: + assert len(report.results) == STORE_KIT.recorded == len(STORE_KIT.cases) + assert all(result.passed for result in (*report.results, *report.envelope_results)) + + +def test_the_manifest_provides_every_capability_from_the_api_package() -> None: + manifest = parse_manifest(files("orvex.store").joinpath("module.toml").read_text(), "store") + (provided,) = manifest.provides + assert provided.contract == STORE.name + assert provided.version == STORE.version + assert set(provided.capabilities) == set(STORE.capabilities) + assert getattr(api, provided.implementation) is Store + assert manifest.requires == () + + +class _ExecutesTwice(Store): + """Planted: an execute that ignores its idempotency key.""" + + def execute( + self, envelope: CallEnvelope, transaction: TransactionHandle, request: StatementRequest + ) -> None: + super().execute(envelope, transaction, request) + super().execute( + CallEnvelope(**{**_fields(envelope), "idempotency_key": None}), transaction, request + ) + + +class _DecimalAsFloat(Store): + """Planted: a read that turns every DECIMAL into a float.""" + + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: + found = super().read_batch(envelope, read) + if found is None: + return None + batch = one_batch(found) + columns = [ + column.cast(pa.float64()) if pa.types.is_decimal(column.type) else column + for column in batch.columns + ] + return batch_bytes(pa.record_batch(columns, names=batch.schema.names)) + + +class _ForgedHandleAccepted(Store): + """Planted: a forged read handle answered as an ended read instead of refused.""" + + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: + if read.id == "read-forged": + return None + return super().read_batch(envelope, read) + + +def _fields(envelope: CallEnvelope) -> dict[str, object]: + return {name: getattr(envelope, name) for name in CallEnvelope.__slots__} + + +@pytest.mark.parametrize( + ("planted", "case"), + [ + (_ExecutesTwice, "execute.idempotent"), + (_DecimalAsFloat, "start-read.exact"), + (_ForgedHandleAccepted, "read-batch.ends"), + ], +) +def test_the_kit_fails_a_planted_broken_guarantee(planted: type[Store], case: str) -> None: + report = run_kit(STORE_KIT, planted, serialized=False) + failed = [problem for problem in kit_problems(STORE_KIT, report) if f": {case} (" in problem] + assert failed, f"the kit did not catch the plant in {case}" diff --git a/packages/store/tests/test_database_sources.py b/packages/store/tests/test_database_sources.py index 768c3d27..9bf1eaf5 100644 --- a/packages/store/tests/test_database_sources.py +++ b/packages/store/tests/test_database_sources.py @@ -16,6 +16,7 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresLakeLocation, @@ -24,7 +25,6 @@ SqliteSourceAttachment, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_existing_lake.py b/packages/store/tests/test_existing_lake.py index 268fa375..95491136 100644 --- a/packages/store/tests/test_existing_lake.py +++ b/packages/store/tests/test_existing_lake.py @@ -17,6 +17,7 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( ExistingLakeAttachment, PostgresLakeLocation, @@ -26,7 +27,6 @@ SqliteLakeLocation, ) from orvex.store import existing_lake -from orvex.store.errors import StoreError from orvex.store.existing_lake import ExistingLake MEMORY = "512MB" diff --git a/packages/store/tests/test_existing_lake_files.py b/packages/store/tests/test_existing_lake_files.py index 2d8bbd8d..9f337ed9 100644 --- a/packages/store/tests/test_existing_lake_files.py +++ b/packages/store/tests/test_existing_lake_files.py @@ -20,6 +20,7 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( CredentialsInUseItem, ExistingLakeAttachment, @@ -31,7 +32,6 @@ SqliteLakeLocation, ) from orvex.store import separation -from orvex.store.errors import StoreError from orvex.store.existing_lake import ExistingLake from orvex.testing.environment import unavailable @@ -235,6 +235,26 @@ def test_a_lake_overlapping_a_protected_location_is_refused_in_both_orders( assert "overlaps a protected location" in error.message or "catalog lies" in error.message +def test_an_overlapping_declaration_is_refused_before_anything_is_attached( + other_device: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # DQ-018, from `conformance/runners/security_engines.py` (#486): the refusal comes before + # DuckLake attaches anything; the runner sees through the contract that nothing is held. + root = other_device / "lakes" + catalog, data = make_lake(root / "owner") + attached: list[str] = [] + real = ExistingLake._attach + + def spied(lake: ExistingLake, target: str, options: list[str]) -> None: + attached.append(target) + real(lake, target, options) + + monkeypatch.setattr(ExistingLake, "_attach", spied) + error = refused_at_attach(attachment(catalog, data, [root / "owner"])) + assert "overlaps a protected location" in error.message + assert attached == [] + + # --- file systems apart ---------------------------------------------------------------------- @@ -519,7 +539,7 @@ def test_the_mount_table_gives_each_mounts_type() -> None: import json, os, subprocess, sys from pathlib import Path from orvex.store import separation -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError root = Path(sys.argv[1]) for name in ("lower", "upper", "work", "merged", "project"): (root / name).mkdir() diff --git a/packages/store/tests/test_existing_lake_reads.py b/packages/store/tests/test_existing_lake_reads.py index 79670f9d..d26f14b2 100644 --- a/packages/store/tests/test_existing_lake_reads.py +++ b/packages/store/tests/test_existing_lake_reads.py @@ -13,6 +13,7 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( AzureStorage, ExistingLakeAttachment, @@ -26,7 +27,6 @@ SqliteLakeLocation, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.existing_lake import ExistingLake from orvex.store.existing_lake_reads import FILE_REFUSED, SNAPSHOT_CHANGED, LakeReads from orvex.store.lake import Limits, open_lake diff --git a/packages/store/tests/test_file_source_exactness.py b/packages/store/tests/test_file_source_exactness.py index 94d94d5f..ffc5faae 100644 --- a/packages/store/tests/test_file_source_exactness.py +++ b/packages/store/tests/test_file_source_exactness.py @@ -21,11 +21,11 @@ import pyarrow as pa import pyarrow.parquet as pq import pytest -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError from orvex.store.file_sources import view_query from orvex.testing.fixtures import frozen -from tests.support import parquet_footer +from . import parquet_footer D = decimal.Decimal UTC = dt.UTC diff --git a/packages/store/tests/test_gcs_azure.py b/packages/store/tests/test_gcs_azure.py index 930a3f60..7956b57b 100644 --- a/packages/store/tests/test_gcs_azure.py +++ b/packages/store/tests/test_gcs_azure.py @@ -19,6 +19,7 @@ import duckdb import pytest from moto.server import ThreadedMotoServer +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( AzureStorage, GcsStorage, @@ -28,7 +29,6 @@ S3Storage, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake from orvex.testing import azurite diff --git a/packages/store/tests/test_json_encoding.py b/packages/store/tests/test_json_encoding.py index a8a4237a..6e503702 100644 --- a/packages/store/tests/test_json_encoding.py +++ b/packages/store/tests/test_json_encoding.py @@ -10,10 +10,11 @@ import pytest from hypothesis import given, settings from hypothesis import strategies as st +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ResultJson from orvex.store.arrow_exact import _with_duckdb_types, exact_select -from orvex.store.errors import StoreError -from orvex.store.json_encoding import arrow_to_json + +from .contract_client import store_client CASES = { "BOOLEAN": ["true", "false"], @@ -62,7 +63,7 @@ def encode(sql: str) -> dict[str, object]: source = relation.to_arrow_reader(1000) schema = _with_duckdb_types(source.schema, types) reader = pa.RecordBatchReader.from_batches(schema, (b.cast(schema) for b in source)) - text = "".join(arrow_to_json(reader)) + text = "".join(store_client().arrow_to_json(reader)) finally: connection.close() document: dict[str, object] = json.loads(text) @@ -191,7 +192,9 @@ def test_unsupported_type_fails_on_the_first_fragment() -> None: relation = connection.sql("SELECT 1 AS ok, [1] AS l") source = relation.to_arrow_reader(10) schema = _with_duckdb_types(source.schema, [str(kind) for kind in relation.types]) - fragments = arrow_to_json(pa.RecordBatchReader.from_batches(schema, iter(()))) + fragments = store_client().arrow_to_json( + pa.RecordBatchReader.from_batches(schema, iter(())) + ) with pytest.raises(StoreError): next(fragments) finally: diff --git a/packages/store/tests/test_lake.py b/packages/store/tests/test_lake.py index c8a7186b..ce15ce74 100644 --- a/packages/store/tests/test_lake.py +++ b/packages/store/tests/test_lake.py @@ -12,12 +12,13 @@ import duckdb import pyarrow as pa import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import LocalLakeLocation, Parameter, ReadRequest, StatementRequest -from orvex.store.errors import StoreError -from orvex.store.json_encoding import arrow_to_json from orvex.store.lake import FOLDER_COST, PATTERN_BUDGET, Lake, Limits, open_lake from orvex.testing.bounds import SAFETY_NET, FolderReads +from .contract_client import store_client + # The shared contract tests run on every catalog, SQLite included (#120). EVERY_CATALOG = True @@ -531,7 +532,7 @@ def test_lake_results_encode_exactly_to_json(lake: Lake) -> None: snapshot=lake.current_snapshot().id, batch_size=16, ) - document = json.loads("".join(arrow_to_json(lake.read(request)))) + document = json.loads("".join(store_client().arrow_to_json(lake.read(request)))) assert document["columns"] == [ {"name": "u", "type": "UHUGEINT"}, {"name": "h", "type": "HUGEINT"}, diff --git a/packages/store/tests/test_lake_configuration.py b/packages/store/tests/test_lake_configuration.py index 40bd1f2d..97421c5b 100644 --- a/packages/store/tests/test_lake_configuration.py +++ b/packages/store/tests/test_lake_configuration.py @@ -10,8 +10,8 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import LocalLakeLocation, ReadRequest, StatementRequest -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_load_arrow.py b/packages/store/tests/test_load_arrow.py index 4d5cb4d5..e03389b2 100644 --- a/packages/store/tests/test_load_arrow.py +++ b/packages/store/tests/test_load_arrow.py @@ -15,8 +15,8 @@ import duckdb import pyarrow as pa import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import StatementRequest -from orvex.store.errors import StoreError from orvex.store.lake import Transaction SCHEMA = pa.schema([("amount", pa.string()), ("n", pa.int64())]) diff --git a/packages/store/tests/test_maintenance.py b/packages/store/tests/test_maintenance.py index cbed027d..08c71ecb 100644 --- a/packages/store/tests/test_maintenance.py +++ b/packages/store/tests/test_maintenance.py @@ -11,6 +11,7 @@ import duckdb import pyarrow as pa import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresLakeLocation, @@ -18,7 +19,6 @@ SqliteLakeLocation, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.lake import Lake # The shared contract tests run on every catalog, SQLite included (#120). diff --git a/packages/store/tests/test_nested_exactness.py b/packages/store/tests/test_nested_exactness.py index 7fa3e460..d6da52dd 100644 --- a/packages/store/tests/test_nested_exactness.py +++ b/packages/store/tests/test_nested_exactness.py @@ -7,8 +7,8 @@ from decimal import Decimal import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ReadRequest -from orvex.store.errors import StoreError from orvex.store.lake import Lake UHUGE_MAX = "340282366920938463463374607431768211455" diff --git a/packages/store/tests/test_object_storage.py b/packages/store/tests/test_object_storage.py index 9ed5dd46..5faa49b8 100644 --- a/packages/store/tests/test_object_storage.py +++ b/packages/store/tests/test_object_storage.py @@ -18,6 +18,7 @@ import duckdb import pytest from moto.server import ThreadedMotoServer +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, ObjectStorage, @@ -26,7 +27,6 @@ SqliteLakeLocation, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.isolation import _sandbox_entries from orvex.store.lake import Lake, Limits, open_lake from orvex.store.sql_guard import _guarded diff --git a/packages/store/tests/test_postgres_catalog.py b/packages/store/tests/test_postgres_catalog.py index 829b9b71..22f99763 100644 --- a/packages/store/tests/test_postgres_catalog.py +++ b/packages/store/tests/test_postgres_catalog.py @@ -9,8 +9,8 @@ from pathlib import Path import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter, PostgresLakeLocation, ReadRequest, StatementRequest -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_read_phases.py b/packages/store/tests/test_read_phases.py index 5847c2ce..f15f939d 100644 --- a/packages/store/tests/test_read_phases.py +++ b/packages/store/tests/test_read_phases.py @@ -16,9 +16,9 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ReadRequest from orvex.store import lake as lake_module -from orvex.store.errors import StoreError from orvex.store.lake import Admission, Guard, Lake, ReadHandle LONG = "SELECT range AS n FROM range(100000000)" diff --git a/packages/store/tests/test_read_scopes.py b/packages/store/tests/test_read_scopes.py index 321f5ccb..950d4c9f 100644 --- a/packages/store/tests/test_read_scopes.py +++ b/packages/store/tests/test_read_scopes.py @@ -10,7 +10,7 @@ from typing import Any import pytest -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError from orvex.store.existing_lake_guard import guarded from orvex.store.read_tree import _read_tree, _walk, pin_tables diff --git a/packages/store/tests/test_redaction.py b/packages/store/tests/test_redaction.py index 258451b0..f6996e22 100644 --- a/packages/store/tests/test_redaction.py +++ b/packages/store/tests/test_redaction.py @@ -6,7 +6,7 @@ import urllib.parse import pytest -from orvex.store.errors import StoreError, redact, register_secret +from orvex.contracts.store.errors import StoreError, redact, register_secret SECRET = "pa'ss\"wo\\rd ü&=" diff --git a/packages/store/tests/test_source_isolation.py b/packages/store/tests/test_source_isolation.py index 6453f8be..89cbe65d 100644 --- a/packages/store/tests/test_source_isolation.py +++ b/packages/store/tests/test_source_isolation.py @@ -17,6 +17,7 @@ import duckdb import pyarrow as pa import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( FileSourceAttachment, LocalLakeLocation, @@ -25,7 +26,6 @@ StatementRequest, ) from orvex.store import source_reader, source_worker -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_source_tls.py b/packages/store/tests/test_source_tls.py index b1dee6fe..641b646b 100644 --- a/packages/store/tests/test_source_tls.py +++ b/packages/store/tests/test_source_tls.py @@ -14,13 +14,13 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, PostgresSourceAttachment, ReadRequest, ) from orvex.store.credentials import _require_source_tls, _source_tls -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_sources.py b/packages/store/tests/test_sources.py index 20718e0f..12e2d080 100644 --- a/packages/store/tests/test_sources.py +++ b/packages/store/tests/test_sources.py @@ -14,6 +14,7 @@ import pyarrow as pa import pyarrow.parquet as pq import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( FileSourceAttachment, LocalLakeLocation, @@ -21,7 +22,6 @@ ReadRequest, StatementRequest, ) -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_statistics.py b/packages/store/tests/test_statistics.py index 96a90155..e3949ebe 100644 --- a/packages/store/tests/test_statistics.py +++ b/packages/store/tests/test_statistics.py @@ -6,13 +6,13 @@ from collections.abc import Iterator import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LakeLocation, StatementRequest, StatisticsRequest, StatisticsResult, ) -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_table_columns.py b/packages/store/tests/test_table_columns.py index 0aa0348c..cf0bc6a2 100644 --- a/packages/store/tests/test_table_columns.py +++ b/packages/store/tests/test_table_columns.py @@ -5,8 +5,8 @@ from collections.abc import Iterator import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import LakeLocation, StatementRequest, TablesRequest -from orvex.store.errors import StoreError from orvex.store.lake import Lake, Limits, open_lake LIMITS = Limits( diff --git a/packages/store/tests/test_tokens.py b/packages/store/tests/test_tokens.py index 85b0ff4a..37314225 100644 --- a/packages/store/tests/test_tokens.py +++ b/packages/store/tests/test_tokens.py @@ -12,8 +12,8 @@ import duckdb import pytest +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter, ParameterType -from orvex.store.errors import StoreError from orvex.store.file_names import FOLDER_COST, PATTERN_BUDGET from orvex.store.parameters import _typed from orvex.store.sql_guard import ( diff --git a/tests/conformance/test_asof_windows.py b/tests/conformance/test_asof_windows.py index 478b3910..3535920f 100644 --- a/tests/conformance/test_asof_windows.py +++ b/tests/conformance/test_asof_windows.py @@ -19,12 +19,12 @@ import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import Spy, ask, corpus_case, lake_for diff --git a/tests/conformance/test_bridges.py b/tests/conformance/test_bridges.py index 77477138..1f85d2b0 100644 --- a/tests/conformance/test_bridges.py +++ b/tests/conformance/test_bridges.py @@ -15,13 +15,13 @@ import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.adapters.duckdb import compile_plan from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import Spy, ask, corpus_case, lake_for diff --git a/tests/conformance/test_clickhouse_adapter.py b/tests/conformance/test_clickhouse_adapter.py index a3c3c854..4fb18c73 100644 --- a/tests/conformance/test_clickhouse_adapter.py +++ b/tests/conformance/test_clickhouse_adapter.py @@ -34,6 +34,8 @@ import pyarrow.ipc import pytest from hypothesis import HealthCheck, given, settings +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter, ParameterType, StatementRequest from orvex.semantic import execution, external from orvex.semantic.adapters import clickhouse as ch @@ -53,8 +55,6 @@ from orvex.semantic.query.binding import Limits as QueryLimits from orvex.semantic.query.binding import bind_query from orvex.semantic.result.descriptors import arrow_type -from orvex.store.errors import StoreError -from orvex.store.lake import Lake from orvex.testing.environment import unavailable from conformance.runners import bi_platform_adapter diff --git a/tests/conformance/test_clickhouse_capabilities.py b/tests/conformance/test_clickhouse_capabilities.py index 843323c8..c28c0022 100644 --- a/tests/conformance/test_clickhouse_capabilities.py +++ b/tests/conformance/test_clickhouse_capabilities.py @@ -18,6 +18,8 @@ import pyarrow as pa import pytest +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter, ReadRequest from orvex.semantic import zonedata from orvex.semantic.adapters import clickhouse as ch @@ -30,9 +32,7 @@ from orvex.semantic.planning import plan from orvex.semantic.query.binding import Limits as QueryLimits from orvex.semantic.query.binding import bind_query -from orvex.store.errors import StoreError -from orvex.store.json_encoding import arrow_to_json -from orvex.store.lake import Lake +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce from tests.conformance import test_asof_windows as asof @@ -390,7 +390,9 @@ def pages(server: base.Server, document: Document, policy: Any = None) -> list[A return found last = reference.select([n for n, _ in duck.sorts]).slice(size - 1, 1) text = "".join( - arrow_to_json(pa.RecordBatchReader.from_batches(last.schema, last.to_batches())) + store().arrow_to_json( + pa.RecordBatchReader.from_batches(last.schema, last.to_batches()) + ) ) start = tuple(json.loads(text)["rows"][0]) diff --git a/tests/conformance/test_clickhouse_engine.py b/tests/conformance/test_clickhouse_engine.py index 63b03302..0b2f935f 100644 --- a/tests/conformance/test_clickhouse_engine.py +++ b/tests/conformance/test_clickhouse_engine.py @@ -1,5 +1,5 @@ """The ClickHouse engine wired as the product wires it (P2-31 PR 4, #225; ADR-0027 §4 to §8): -the store's verified read-only connection (`orvex.store.clickhouse`), the server's adapter +the store's verified read-only connection (the store contract's `clickhouse`), the server's adapter (`orvex.server.clickhouse_adapter`) and its evidence schedule (`clickhouse_evidence`), asked through `orvex.semantic.execution.answer` as `server` asks. @@ -24,6 +24,9 @@ import pytest import yaml +from orvex.contracts.store.client import Lake +from orvex.contracts.store.client_sources import ClickHouse, connect_clickhouse +from orvex.contracts.store.values import Budget, ClickHouseTarget from orvex.semantic import execution from orvex.semantic.adapters import clickhouse as ch from orvex.semantic.adapters.clickhouse import CAPABILITIES as CLICKHOUSE_CAPABILITIES @@ -41,8 +44,7 @@ from orvex.server.clickhouse_adapter import ClickHouseAdapter from orvex.server.clickhouse_engines import tables from orvex.server.clickhouse_evidence import EvidenceSchedule, checked -from orvex.store.clickhouse import Budget, ClickHouse, ClickHouseTarget -from orvex.store.lake import Lake +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce from tests.conformance import test_clickhouse_adapter as base @@ -69,7 +71,7 @@ def engine(server: base.Server, monkeypatch: pytest.MonkeyPatch) -> ClickHouse: user, password = server.reader # type: ignore[attr-defined] monkeypatch.setenv(PASSWORD, password) target = ClickHouseTarget(server.host, server.port, server.database, user, PASSWORD, tls=False) - return ClickHouse.connect(target, BUDGET) + return connect_clickhouse(store(), target, BUDGET) def routing_of(compiled: CompiledModel) -> Routing: diff --git a/tests/conformance/test_conformed.py b/tests/conformance/test_conformed.py index c8239a53..edff5368 100644 --- a/tests/conformance/test_conformed.py +++ b/tests/conformance/test_conformed.py @@ -16,13 +16,13 @@ import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.adapters.duckdb import compile_plan from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import Spy, ask, corpus_case, lake_for diff --git a/tests/conformance/test_detail_rows.py b/tests/conformance/test_detail_rows.py index 077a4260..4e03e50f 100644 --- a/tests/conformance/test_detail_rows.py +++ b/tests/conformance/test_detail_rows.py @@ -15,6 +15,7 @@ import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import StatementRequest from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import compile_model @@ -23,7 +24,6 @@ from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.result.cursors import SealedForm -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance import test_conformed as conformed diff --git a/tests/conformance/test_direct_query.py b/tests/conformance/test_direct_query.py index 722bf3c5..ebefc648 100644 --- a/tests/conformance/test_direct_query.py +++ b/tests/conformance/test_direct_query.py @@ -18,6 +18,7 @@ import pyarrow as pa import pytest import yaml +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import Parameter, ReadRequest, StatementRequest from orvex.semantic.adapters.duckdb import PhysicalQuery, PolicyValues, compile_plan from orvex.semantic.compiler.compiled import compile_model @@ -33,7 +34,6 @@ from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.receipt.viewer import CallerViewer -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance import test_row_policies as rp diff --git a/tests/conformance/test_estimates.py b/tests/conformance/test_estimates.py index 549c23e4..055f4c06 100644 --- a/tests/conformance/test_estimates.py +++ b/tests/conformance/test_estimates.py @@ -12,10 +12,10 @@ import pytest from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import StatementRequest, StatisticsRequest, StatisticsResult from orvex.semantic.estimates import estimate -from orvex.store.errors import StoreError -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance import test_detail_rows as detail diff --git a/tests/conformance/test_evidence.py b/tests/conformance/test_evidence.py index 0aec65e0..cfc2bb07 100644 --- a/tests/conformance/test_evidence.py +++ b/tests/conformance/test_evidence.py @@ -13,6 +13,8 @@ from typing import Any import pytest +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ChangesRequest, StatementRequest from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model @@ -22,8 +24,6 @@ from orvex.semantic.ossie.reader import read from orvex.semantic.planning import plan from orvex.semantic.query.binding import Limits, bind_query -from orvex.store.errors import StoreError -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance import test_materialization as rollups diff --git a/tests/conformance/test_exact_arithmetic.py b/tests/conformance/test_exact_arithmetic.py index 6b9c7d62..c4714d68 100644 --- a/tests/conformance/test_exact_arithmetic.py +++ b/tests/conformance/test_exact_arithmetic.py @@ -14,6 +14,7 @@ import duckdb import pytest import yaml +from orvex.contracts.store.client import Lake from orvex.semantic import SemanticError from orvex.semantic.adapters import duckdb as duckdb_adapter from orvex.semantic.compiler.catalog import SourceColumn @@ -27,7 +28,6 @@ from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.query.binding import Limits as QueryLimits from orvex.semantic.query.binding import bind_query -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.support.policies import VENDORS, extension diff --git a/tests/conformance/test_execution.py b/tests/conformance/test_execution.py index 838c7fd9..4fa70bcd 100644 --- a/tests/conformance/test_execution.py +++ b/tests/conformance/test_execution.py @@ -13,6 +13,8 @@ import pyarrow as pa import pytest import yaml +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import StatementRequest from orvex.semantic.adapters import clickhouse from orvex.semantic.adapters import duckdb as adapter_module @@ -23,9 +25,7 @@ from orvex.semantic.ossie.reader import read from orvex.semantic.planning import SemanticPlan, plan from orvex.semantic.policies import PolicyModel, read_policies -from orvex.store.errors import StoreError -from orvex.store.json_encoding import arrow_to_json -from orvex.store.lake import Lake +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce from conformance.runners.semantic_corpus import ROOT, run @@ -391,7 +391,9 @@ def test_two_runs_share_the_execution_identity_but_not_necessarily_the_receipt() three = ask(REVENUE_BY_REGION, compiled, lake) identity = [a.receipt.document()["execution_fingerprint"] for a in (one, two, three)] assert identity[0] == identity[1] != identity[2] - assert one.result.canonical(arrow_to_json) == two.result.canonical(arrow_to_json) + assert one.result.canonical(store().arrow_to_json) == two.result.canonical( + store().arrow_to_json + ) assert one.receipt.receipt_id != two.receipt.receipt_id # timings differ assert one.plan.fingerprint == two.plan.fingerprint diff --git a/tests/conformance/test_existing_ducklake.py b/tests/conformance/test_existing_ducklake.py index 89279aa3..441c9136 100644 --- a/tests/conformance/test_existing_ducklake.py +++ b/tests/conformance/test_existing_ducklake.py @@ -8,6 +8,7 @@ from __future__ import annotations +import contextlib import datetime as dt import json import shutil @@ -21,12 +22,15 @@ import duckdb import jsonschema import pytest +from orvex.contracts.store.client_sources import ExistingLakeReads, open_existing_lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( ExistingLakeAttachment, ProtectedLocation, Snapshot, SqliteLakeLocation, ) +from orvex.contracts.store.values import ExistingLakeLimits from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.engines import EngineEntry, Routing from orvex.semantic.errors import SemanticError @@ -38,9 +42,7 @@ from orvex.server.existing_lake_adapter import ExistingLakeAdapter from orvex.server.existing_lake_evidence import violated from orvex.server.existing_lake_snapshots import SnapshotSchedule -from orvex.store.errors import StoreError -from orvex.store.existing_lake import ExistingLake -from orvex.store.existing_lake_reads import LakeReads +from orvex.server.store_access import store from orvex.testing.environment import unavailable from conformance.runners import corpus_execution as ce @@ -97,11 +99,14 @@ def edit_catalog(self, sql: str, *parameters: object) -> None: connection.close() +LIMITS = ExistingLakeLimits(memory_limit="512MB", threads=1, timeout_ms=60_000) + + @dataclass class Engine: owner: Owner - lake: ExistingLake - reads: LakeReads + lake: ExistingLakeReads + reads: ExistingLakeReads schedule: SnapshotSchedule adapter: ExistingLakeAdapter attachment: ExistingLakeAttachment @@ -150,8 +155,7 @@ def make_attachment(owner: Owner, protected: list[str]) -> ExistingLakeAttachmen @pytest.fixture def engine(owner: Owner, tmp_path: Path) -> Iterator[Engine]: made = make_attachment(owner, [str(tmp_path / "project")]) - lake = ExistingLake(made, memory_limit="512MB", threads=1) - reads = LakeReads(lake, timeout_ms=60_000) + lake = reads = open_existing_lake(store(), made, LIMITS) holds: list[str] = [] current = {"attachment": made} schedule = SnapshotSchedule( @@ -391,8 +395,7 @@ def test_a_snapshot_with_violated_evidence_is_held_back_and_the_previous_one_ser made = make_attachment(owner, [str(tmp_path / "project")]) routing = routed(("sales", ENTRY), ("customers", ENTRY)) compiled = evidenced() - with ExistingLake(made, memory_limit="512MB", threads=1) as lake: - reads = LakeReads(lake, timeout_ms=60_000) + with contextlib.closing(open_existing_lake(store(), made, LIMITS)) as reads: def evidence(snapshot: Snapshot) -> str | None: return violated( @@ -448,8 +451,7 @@ def test_a_snapshot_whose_files_fail_is_held_back(engine: Engine, tmp_path: Path def test_a_protected_location_registered_later_stops_serving(owner: Owner, tmp_path: Path) -> None: made = make_attachment(owner, [str(tmp_path / "project")]) current = {"attachment": made} - with ExistingLake(made, memory_limit="512MB", threads=1) as lake: - reads = LakeReads(lake, timeout_ms=60_000) + with contextlib.closing(open_existing_lake(store(), made, LIMITS)) as reads: schedule = SnapshotSchedule( PROJECT, "warehouse", diff --git a/tests/conformance/test_explain.py b/tests/conformance/test_explain.py index 2d870997..cb1fcb8b 100644 --- a/tests/conformance/test_explain.py +++ b/tests/conformance/test_explain.py @@ -13,6 +13,7 @@ import pytest import yaml from fastapi.testclient import TestClient +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import ChangesRequest, ReadRequest from orvex.semantic.compiler.compiled import CompiledModel from orvex.semantic.errors import SemanticError @@ -25,7 +26,6 @@ from orvex.server.app import LocalSemantic, create_app from orvex.server.local_access import new_access from orvex.server.versions import Versions -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from conformance.runners.semantic_corpus import CORPUS, ROOT diff --git a/tests/conformance/test_fanout_isolation.py b/tests/conformance/test_fanout_isolation.py index e2687f06..2a250f6a 100644 --- a/tests/conformance/test_fanout_isolation.py +++ b/tests/conformance/test_fanout_isolation.py @@ -15,13 +15,13 @@ import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.adapters.duckdb import compile_plan from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import Spy, ask, corpus_case, lake_for diff --git a/tests/conformance/test_paths_roles.py b/tests/conformance/test_paths_roles.py index fcb089d6..e4841225 100644 --- a/tests/conformance/test_paths_roles.py +++ b/tests/conformance/test_paths_roles.py @@ -14,6 +14,7 @@ import yaml from hypothesis import given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.correctness import analyze @@ -22,7 +23,6 @@ from orvex.semantic.ossie.reader import read from orvex.semantic.policies import read_policies from orvex.semantic.query.binding import Limits, bind_query -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import ask, corpus_case, lake_for diff --git a/tests/conformance/test_period_end.py b/tests/conformance/test_period_end.py index 3d59dc95..f9c8855f 100644 --- a/tests/conformance/test_period_end.py +++ b/tests/conformance/test_period_end.py @@ -15,7 +15,7 @@ from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read -from orvex.store.json_encoding import arrow_to_json +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import ask, corpus_case, lake_for, tiny_lake @@ -245,7 +245,9 @@ def test_the_physical_order_of_rows_changes_nothing() -> None: one = answer_of(compiled, catalog, data, query("balance", group_by=BY_MONTH)) two = answer_of(compiled, catalog, shuffled, query("balance", group_by=BY_MONTH)) assert ce.rows_of(one) == ce.rows_of(two) == [["N", "2026-01-01", "168.00"]] - assert one.result.canonical(arrow_to_json) == two.result.canonical(arrow_to_json) + assert one.result.canonical(store().arrow_to_json) == two.result.canonical( + store().arrow_to_json + ) def test_a_key_wider_than_entity_and_axis_is_refused_even_without_ties() -> None: diff --git a/tests/conformance/test_ratios.py b/tests/conformance/test_ratios.py index b7e1797b..3dd88efc 100644 --- a/tests/conformance/test_ratios.py +++ b/tests/conformance/test_ratios.py @@ -9,11 +9,11 @@ import pytest from hypothesis import given, settings from hypothesis import strategies as st +from orvex.contracts.store.errors import StoreError from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read -from orvex.store.errors import StoreError from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import ask, corpus_case, lake_for, tiny_lake diff --git a/tests/conformance/test_receipt_result.py b/tests/conformance/test_receipt_result.py index 16d4bd9c..9be919e2 100644 --- a/tests/conformance/test_receipt_result.py +++ b/tests/conformance/test_receipt_result.py @@ -9,7 +9,7 @@ from orvex.semantic.canonical import fingerprint from orvex.semantic.receipt import verify_receipt, verify_replay from orvex.semantic.result import InvalidResult, SemanticResult, build_result, read_result -from orvex.store.json_encoding import arrow_to_json +from orvex.server.store_access import store from tests.support.receipts import EVERYTHING, QUERY, RESULT, SEAL, SampleViewer, receipt from tests.support.semantic_results import MODEL, top_n_with_others @@ -42,34 +42,40 @@ def result_for_query() -> SemanticResult: def test_a_result_references_its_receipt_and_its_replay_verifies() -> None: result = result_for_query() - canonical = result.canonical(arrow_to_json) + canonical = result.canonical(store().arrow_to_json) record = receipt(result_canonical=canonical) referenced = result.with_receipt(record.receipt_id, record.reproducibility["class"]) - assert referenced.canonical(arrow_to_json) == canonical # the reference is not hashed + assert referenced.canonical(store().arrow_to_json) == canonical # the reference is not hashed viewer = SampleViewer() view = record.view(viewer) assert verify_receipt(view, record, SEAL, viewer) - document = referenced.wire(arrow_to_json) + document = referenced.wire(store().arrow_to_json) assert read_result(document, referenced.arrow(), view).envelope["receipt"] == { "receipt_id": record.receipt_id, "reproducibility": "exact", } - replayed = verify_replay(record, result_for_query().canonical(arrow_to_json), RESULT, SEAL) + replayed = verify_replay( + record, result_for_query().canonical(store().arrow_to_json), RESULT, SEAL + ) assert replayed.view(EVERYTHING)["reproducibility"]["replay"] == "verified" def test_a_result_that_names_another_receipt_is_refused() -> None: result = result_for_query() - record = receipt(result_canonical=result.canonical(arrow_to_json)) + record = receipt(result_canonical=result.canonical(store().arrow_to_json)) view = record.view(SampleViewer()) other = receipt(result_canonical=b'{"other":1}') with pytest.raises(InvalidResult, match="receipt"): - read_result(result.with_receipt(other.receipt_id, "exact").wire(arrow_to_json), None, view) - without = result.wire(arrow_to_json) + read_result( + result.with_receipt(other.receipt_id, "exact").wire(store().arrow_to_json), None, view + ) + without = result.wire(store().arrow_to_json) with pytest.raises(InvalidResult, match="receipt"): read_result(without, None, view) assert read_result(without).envelope.get("receipt") is None # still valid without one - foreign = top_n_with_others().with_receipt(record.receipt_id, "exact").wire(arrow_to_json) + foreign = ( + top_n_with_others().with_receipt(record.receipt_id, "exact").wire(store().arrow_to_json) + ) with pytest.raises(InvalidResult, match="receipt"): read_result(foreign, None, view) # another query's result assert json.loads(foreign)["query_fingerprint"] != QUERY_FINGERPRINT diff --git a/tests/conformance/test_result_cache.py b/tests/conformance/test_result_cache.py index a379b212..4625d76a 100644 --- a/tests/conformance/test_result_cache.py +++ b/tests/conformance/test_result_cache.py @@ -22,6 +22,8 @@ from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st from orvex.access.context import permission_fingerprint +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import StatementRequest from orvex.semantic.errors import SemanticError from orvex.semantic.execution import Answer, answer @@ -29,8 +31,6 @@ from orvex.semantic.receipt import verify_receipt from orvex.semantic.receipt.viewer import CallerViewer from orvex.semantic.result_cache import ResultCache -from orvex.store.errors import StoreError -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from conformance.runners import security_fixtures as sf diff --git a/tests/conformance/test_row_policies.py b/tests/conformance/test_row_policies.py index e7900d57..043c5986 100644 --- a/tests/conformance/test_row_policies.py +++ b/tests/conformance/test_row_policies.py @@ -15,13 +15,13 @@ import yaml from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import StatementRequest from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read from orvex.semantic.policies import InvalidContext, PolicyModel, read_policies -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance import test_asof_windows as windows diff --git a/tests/conformance/test_schema_sources.py b/tests/conformance/test_schema_sources.py index a891b605..1044478c 100644 --- a/tests/conformance/test_schema_sources.py +++ b/tests/conformance/test_schema_sources.py @@ -13,11 +13,11 @@ from typing import Any, cast import pytest +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import StatisticsRequest, StatisticsResult from orvex.semantic.errors import SemanticError from orvex.semantic.evidence import verify -from orvex.store.errors import StoreError -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from conformance.runners.semantic_corpus import CORPUS, ROOT diff --git a/tests/conformance/test_semantic_result.py b/tests/conformance/test_semantic_result.py index 60fe7ae5..44e4fa4b 100644 --- a/tests/conformance/test_semantic_result.py +++ b/tests/conformance/test_semantic_result.py @@ -21,7 +21,7 @@ from hypothesis import strategies as st from orvex.semantic import SemanticError from orvex.semantic.result import CellState, SemanticResult, read_result -from orvex.store.json_encoding import arrow_to_json +from orvex.server.store_access import store from tests.support.semantic_results import ( decimal as decimal_column, @@ -44,7 +44,7 @@ def wire(result: SemanticResult) -> dict[str, Any]: - document: dict[str, Any] = json.loads(result.wire(arrow_to_json)) + document: dict[str, Any] = json.loads(result.wire(store().arrow_to_json)) return document @@ -104,7 +104,7 @@ def test_decimals_of_every_precision_and_scale_stay_exact( assert isinstance(cell, str) assert decimal.Decimal(cell) == value assert len(cell.partition(".")[2]) == scale - canonical = json.loads(result.canonical(arrow_to_json)) + canonical = json.loads(result.canonical(store().arrow_to_json)) assert canonical["rows"] == [[cell], [None]] @@ -168,7 +168,7 @@ def test_the_canonical_form_keeps_every_value_exact() -> None: ] result = detail(source, columns) assert wire(result)["rows"][0] == [-0.0, 0.1, 1] - canonical = result.canonical(arrow_to_json) + canonical = result.canonical(store().arrow_to_json) rows = json.loads(canonical)["rows"] assert rows == [ ["-0.0", "0.1", "1"], @@ -177,19 +177,18 @@ def test_the_canonical_form_keeps_every_value_exact() -> None: ["-Infinity", None, "4"], ["0.1", "1.5", "-5"], ] - exact = detail(exact_table(), EXACT_COLUMNS).canonical(arrow_to_json) + exact = detail(exact_table(), EXACT_COLUMNS).canonical(store().arrow_to_json) assert json.loads(exact)["rows"] == [[DECIMAL_38_10, HUGEINT_MIN, UBIGINT_MAX, UHUGEINT_MAX]] CANONICAL_SCRIPT = """ import hashlib, sys sys.path.insert(0, {root!r}) -from orvex.store.json_encoding import arrow_to_json from tests.conformance.test_semantic_result import detail, exact_table, EXACT_COLUMNS from tests.support.semantic_results import top_n_with_others parts = [ - top_n_with_others().canonical(arrow_to_json), - detail(exact_table(), EXACT_COLUMNS).canonical(arrow_to_json), + top_n_with_others().canonical(store().arrow_to_json), + detail(exact_table(), EXACT_COLUMNS).canonical(store().arrow_to_json), ] print(hashlib.sha256(b"|".join(parts)).hexdigest()) """ @@ -197,8 +196,8 @@ def test_the_canonical_form_keeps_every_value_exact() -> None: def canonical_digest() -> str: parts = [ - top_n_with_others().canonical(arrow_to_json), - detail(exact_table(), EXACT_COLUMNS).canonical(arrow_to_json), + top_n_with_others().canonical(store().arrow_to_json), + detail(exact_table(), EXACT_COLUMNS).canonical(store().arrow_to_json), ] return hashlib.sha256(b"|".join(parts)).hexdigest() @@ -233,6 +232,9 @@ def test_an_empty_page_with_a_masked_column_and_a_notice_reveals_no_builder_deta assert document["rows"] == [] assert document["row_count"] == 0 assert [n["code"] for n in document["notices"]] == ["FILTER_INTERSECTION_EMPTY"] - for form in (result.wire(arrow_to_json).encode(), result.canonical(arrow_to_json)): + for form in ( + result.wire(store().arrow_to_json).encode(), + result.canonical(store().arrow_to_json), + ): assert b"consumers.region" not in form assert read_result(json.dumps(document), result.arrow()).cells == () diff --git a/tests/conformance/test_zones.py b/tests/conformance/test_zones.py index fcebcc58..a32c7d87 100644 --- a/tests/conformance/test_zones.py +++ b/tests/conformance/test_zones.py @@ -18,12 +18,12 @@ import pytest from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic import zonedata from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError from orvex.semantic.ossie.reader import read -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_execution import ask, corpus_case, lake_for diff --git a/tests/properties/exact.py b/tests/properties/exact.py index 6d3d6758..8f681091 100644 --- a/tests/properties/exact.py +++ b/tests/properties/exact.py @@ -22,10 +22,10 @@ import duckdb from hypothesis import settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.ossie.reader import read -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.properties.world import NIGHTLY, PROFILE diff --git a/tests/properties/policy_engine.py b/tests/properties/policy_engine.py index 23631884..89b90dff 100644 --- a/tests/properties/policy_engine.py +++ b/tests/properties/policy_engine.py @@ -14,11 +14,11 @@ from typing import Any import yaml +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.execution import Answer, answer from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.result.cursors import SealedForm -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from conformance.runners.security_engines import column_name diff --git a/tests/properties/test_engine_oracle.py b/tests/properties/test_engine_oracle.py index 27ad28e3..ca473d6a 100644 --- a/tests/properties/test_engine_oracle.py +++ b/tests/properties/test_engine_oracle.py @@ -175,12 +175,14 @@ def test_detail_pages_concatenate_to_every_row_exactly_once_in_order( def test_the_same_question_twice_is_the_same_answer(world: World, query: Document) -> None: """Semantic determinism (Owner correction 6): equal canonical results and equal execution fingerprints; volatile metadata (receipt ids, timings) is outside both.""" - from orvex.store.json_encoding import arrow_to_json + from orvex.server.store_access import store with lake_of(world) as lake: first = ask(query, lake) second = ask(query, lake) - assert first.result.canonical(arrow_to_json) == second.result.canonical(arrow_to_json) + assert first.result.canonical(store().arrow_to_json) == second.result.canonical( + store().arrow_to_json + ) one = first.receipt.document() two = second.receipt.document() assert one["execution_fingerprint"] == two["execution_fingerprint"] @@ -191,13 +193,13 @@ def test_the_same_question_twice_is_the_same_answer(world: World, query: Documen @given(worlds(), aggregate_queries()) def test_arrow_and_json_are_the_same_generated_answer(world: World, query: Document) -> None: """Owner correction 3 (HTTP): the #363 equivalence, on generated answers.""" - from orvex.store.json_encoding import arrow_to_json + from orvex.server.store_access import store with lake_of(world) as lake: found = ask(query, lake) context = {**ce.viewer_context(), "role": "builder"} receipt = found.receipt.view(CallerViewer(COMPILED, NO_POLICIES, context)) - document = {"result": json.loads(found.result.wire(arrow_to_json)), "receipt": receipt} + document = {"result": json.loads(found.result.wire(store().arrow_to_json)), "receipt": receipt} table = arrow_answer(found.result, receipt) sink = pa.BufferOutputStream() with pa.ipc.new_stream(sink, table.schema) as writer: diff --git a/tests/properties/test_exact_properties.py b/tests/properties/test_exact_properties.py index 050585ac..15a8ca3d 100644 --- a/tests/properties/test_exact_properties.py +++ b/tests/properties/test_exact_properties.py @@ -17,10 +17,10 @@ import yaml from hypothesis import example, given from hypothesis import strategies as st +from orvex.contracts.store.errors import StoreError from orvex.semantic import SemanticError from orvex.semantic.execution import answer from orvex.semantic.policies import read_policies -from orvex.store.errors import StoreError from conformance.runners import corpus_execution as ce from tests.properties import exact diff --git a/tests/properties/test_hidden_key_inference.py b/tests/properties/test_hidden_key_inference.py index 7298966e..0c21a09f 100644 --- a/tests/properties/test_hidden_key_inference.py +++ b/tests/properties/test_hidden_key_inference.py @@ -30,11 +30,11 @@ import yaml from hypothesis import given from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.errors import SemanticError from orvex.semantic.execution import Answer, answer from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.receipt.viewer import CallerViewer -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.conformance.test_period_end import snapshots_model diff --git a/tests/properties/test_parser_fuzzing.py b/tests/properties/test_parser_fuzzing.py index 03ac75eb..4f2a172e 100644 --- a/tests/properties/test_parser_fuzzing.py +++ b/tests/properties/test_parser_fuzzing.py @@ -1,17 +1,10 @@ -"""P3-14 (#448; #417): the two parsers that read text from users, fuzzed. +"""P3-14 (#448; #417): the project's SQL parser, which reads text from users, fuzzed (the +ClickHouse statement scanner's half is the store's own, `test_clickhouse_scanner_fuzzing.py`). - **The project's SQL parser** (`orvex.semantic.ossie.sql.parse`). On grammar-guided expressions, then mutated, it returns a tree or raises `SqlSyntaxError` or `SqlUnsupported`, never anything else; it is deterministic; whitespace between tokens never changes the tree; nesting is bounded (`MAX_DEPTH`) and a long input ends fast. -- **The ClickHouse statement scanner** (`orvex.store.clickhouse._bound`). On statements built - from the adapter's own pieces and the lexer's hard cases (strings and their escapes, - heredocs, quoted names, every comment form, placeholders, `;`, `FORMAT`, `SETTINGS`), it - accepts or refuses with `STORE_STATEMENT_INVALID` or `STORE_PARAMETER_INVALID`. Every - statement it accepts is checked against `clickhouse_view`, a lexer written here from - ClickHouse's documented syntax, independently of the scanner: what ClickHouse would run is - one `SELECT` (or `WITH`), with no `;` and no `FORMAT`, `SETTINGS` or `INTO OUTFILE`. A - scanner that agrees only with itself proves nothing. """ from __future__ import annotations @@ -21,10 +14,7 @@ from hypothesis import example, given from hypothesis import strategies as st -from orvex.contracts.store.v1 import Parameter from orvex.semantic.ossie import sql -from orvex.store import clickhouse -from orvex.store.errors import StoreError from tests.properties.world import PROFILE @@ -96,147 +86,3 @@ def test_the_sql_parser_returns_a_tree_or_refuses_and_ends_fast(text: str) -> No def test_whitespace_between_tokens_never_changes_the_tree(text: str) -> None: spaced = re.sub(r"([(),])", r" \1\n ", text) # tokens never part of another assert outcome(spaced) == outcome(text) - - -# --- the ClickHouse scanner ------------------------------------------------------------------ - -PIECES = ( - "SELECT", "WITH x AS (SELECT 1)", "(SELECT", ")", "`t`.`amount`", '"t"."zone"', "x", "1", - ",", "FROM `db`.`t`", "WHERE", "=", "AND", "GROUP BY 1", "ORDER BY 1", "LIMIT 10", - "{p0:String}", "{p1:Nullable(String)}", "{p2:Int64}", "'a'", "'it''s'", "'\\''", - "'\\\\'", "$$ raw ; FORMAT $$", "$tag$ x $tag$", "`we``ird`", '"a\\"b"', - "-- line ; FORMAT\n", "# hash SETTINGS a=1\n", "#! bang ;\n", "#x\n", "// slashes ;\n", - "/* c ; */", "/* /* nested */ SETTINGS x = 1 */", "/* /* */ '", "' */", "' -- '", "*/", - "/* open", "'open", "`open", "$x$ open", - ";", "FORMAT JSON", "SETTINGS max_threads = 1", "INTO OUTFILE 'f'", "format", "\n", " ", - "Settings", "formatDateTime(x, '%F')", "settingsx", "FORMAT_x", -) # fmt: skip -TYPES = {"p0": "String", "p1": "Nullable(String)", "p2": "Int64"} - - -@st.composite -def statements(draw: st.DrawFn) -> str: - start = draw(st.sampled_from(("SELECT", "WITH x AS (SELECT 1) SELECT", "", "(SELECT"))) - rest = draw(st.lists(st.sampled_from(PIECES), max_size=10)) - glue = draw(st.sampled_from((" ", "", "\n"))) - return glue.join([start, *rest]).strip() - - -def parameters(statement: str) -> list[Parameter]: - """One parameter for each placeholder name written anywhere, as the adapter would bind it.""" - named = sorted(set(re.findall(r"\{(p[0-9]):", statement))) - values = {"p0": ("VARCHAR", "a"), "p1": ("VARCHAR", None), "p2": ("BIGINT", "7")} - return [ - Parameter.model_validate({"name": n, "type": values[n][0], "value": values[n][1]}) - for n in named - ] - - -def clickhouse_view(text: str) -> str | None: - """What ClickHouse's lexer leaves as code, from its documented syntax (written here, apart - from the scanner): strings `'…'` with `\\` escapes and `''`; heredocs `$tag$…$tag$`; - names in backquotes or double quotes with `\\` escapes and doubled quotes; comments `--`, - `#!`, `# ` and `//` to the end of the line, and `/* … */`, which nest. Each string and name - becomes `''` or `x`, each comment a space. None when the lexer would refuse the text: an - unclosed quote or comment, or a `#` that starts no comment.""" - code, at = [], 0 - while at < len(text): - char, two = text[at], text[at : at + 2] - if two in {"--", "//", "#!", "# "}: - end = text.find("\n", at) - at = len(text) if end < 0 else end - code.append(" ") - elif char == "#": - return None - elif two == "/*": - depth, at = 1, at + 2 - while depth: - if at >= len(text): - return None - if text.startswith("/*", at): - depth, at = depth + 1, at + 2 - elif text.startswith("*/", at): - depth, at = depth - 1, at + 2 - else: - at += 1 - code.append(" ") - elif char == "$" and (tag := re.match(r"\$(\w*)\$", text[at:])): - close = text.find(tag.group(0), at + len(tag.group(0))) - if close < 0: - return None - at = close + len(tag.group(0)) - code.append(" '' ") - elif char in "'`\"": - at += 1 - while True: - if at >= len(text): - return None - if text[at] == "\\" or (text[at] == char and text[at + 1 : at + 2] == char): - at += 2 - elif text[at] == char: - at += 1 - break - else: - at += 1 - code.append(" '' " if char == "'" else " x ") - else: - code.append(char) - at += 1 - return "".join(code) - - -PLACEHOLDER = r"\{([A-Za-z_][A-Za-z0-9_]*):([^{}]+)\}" - - -def runs_as_one_select(code: str) -> bool: - plain = re.sub(PLACEHOLDER, " 0 ", code).strip() - return ( - ";" not in plain - and re.match(r"(?is)^\(*\s*(SELECT|WITH)\b", plain) is not None - and re.search(r"(?i)\b(SETTINGS|FORMAT|INTO\s+OUTFILE)\b", plain) is None - ) - - -@PROFILE -@example("SELECT 1 /* /* */ SETTINGS max_threads = 1 */") -@example("SELECT 1 // ; FORMAT JSON\n") -@example("SELECT 1 #SETTINGS max_threads = 1") -@example("SELECT 1 $$ ; $$ FORMAT JSON") -@example("SELECT '\\'' ; SELECT 2") -# Found by the Reviewer pass on #448: a nested comment made the scanner read code as a string. -@example("SELECT 1 /* /* */ ' */ SETTINGS max_execution_time = 0 -- '") -@given(statements()) -def test_what_the_scanner_accepts_runs_on_clickhouse_as_one_plain_select(statement: str) -> None: - try: - clickhouse._bound(statement, parameters(statement)) - except StoreError as error: - refused: str | None = error.code - else: - refused = None - if refused is not None: - assert refused in {"STORE_STATEMENT_INVALID", "STORE_PARAMETER_INVALID"} - return - code = clickhouse_view(statement) - assert code is None or runs_as_one_select(code), code - if code is not None: # each placeholder found once, with its written type, where it runs - written = [(n, t.strip()) for n, t in re.findall(PLACEHOLDER, code)] - assert clickhouse._scan(statement)[1] == written - - -def test_the_reference_lexer_sees_what_clickhouse_documents() -> None: - """The checks above are only as good as this lexer: each documented case, and each - property, fails on a statement built to break it.""" - assert clickhouse_view("SELECT 1 -- ; x\n") == "SELECT 1 \n" - assert clickhouse_view("SELECT /* a /* b */ c */ 1") == "SELECT 1" - assert clickhouse_view("SELECT 'a\\'b'") == "SELECT '' " - assert clickhouse_view("SELECT $t$ ; $t$") == "SELECT '' " - assert clickhouse_view("SELECT 'open") is None - assert clickhouse_view("SELECT 1 #x") is None - assert runs_as_one_select("SELECT 1") - for broken in ( - "SELECT 1; SELECT 2", - "SELECT 1 FORMAT JSON", - "DROP TABLE t", - "SELECT 1 settings a = 1", - ): - assert not runs_as_one_select(broken), broken diff --git a/tests/properties/world.py b/tests/properties/world.py index 257d031c..d19da47a 100644 --- a/tests/properties/world.py +++ b/tests/properties/world.py @@ -21,13 +21,13 @@ from hypothesis import HealthCheck, settings from hypothesis import strategies as st +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.execution import Answer, answer from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.result.cursors import SealedForm -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce from tests.support.policies import VENDORS, extension From 979e239ed19ac93d2c2dd3ef72b8aa4c01f4534b Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 02:51:25 -0400 Subject: [PATCH 14/21] refactor(store): consumers reach the store only through its interface contract `orvex.contracts.store`: the interface contract (every operation consumers use, by capability), its kit (58 cases, no skips), its errors with their codes and builders, its values and handles, its Arrow IPC transport and its consumer client. `orvex.store.api.Store` implements it, with `module.toml`. `server`, the runners, `tools` and the repository tests call the contract; the boundaries report shows no store violation. The standalone store job is added. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .github/workflows/interfaces.yml | 112 + .github/workflows/property-nightly.yml | 2 +- benchmarks/source-stream.md | 2 +- conformance/runners/corpus_clickhouse.py | 11 +- conformance/runners/corpus_execution.py | 13 +- .../runners/corpus_existing_ducklake.py | 12 +- conformance/runners/corpus_http.py | 6 +- conformance/runners/security_engines.py | 52 +- conformance/runners/security_fixtures.py | 2 +- conformance/runners/security_http.py | 2 +- packages/contracts/interface-record.json | 2814 ++++++++++++++++- .../src/orvex/contracts/store/arrow_ipc.py | 37 + .../src/orvex/contracts/store/client.py | 316 ++ .../orvex/contracts/store/client_sources.py | 147 + .../src/orvex/contracts/store/interface.py | 394 +++ .../src/orvex/contracts/store/kit.py | 910 ++++++ .../src/orvex/contracts/store/kit_support.py | 177 ++ .../src/orvex/contracts/store/values.py | 137 + packages/server/src/orvex/server/app.py | 2 +- packages/server/src/orvex/server/assembly.py | 36 + packages/server/src/orvex/server/cli.py | 2 +- .../src/orvex/server/clickhouse_adapter.py | 2 +- .../src/orvex/server/clickhouse_engines.py | 29 +- .../src/orvex/server/clickhouse_evidence.py | 2 +- .../server/src/orvex/server/error_problems.py | 2 +- .../src/orvex/server/evidence_command.py | 4 +- .../src/orvex/server/evidence_records.py | 4 +- .../server/src/orvex/server/evidence_runs.py | 4 +- .../src/orvex/server/existing_lake_adapter.py | 6 +- .../orvex/server/existing_lake_snapshots.py | 4 +- .../server/src/orvex/server/existing_lakes.py | 27 +- packages/server/src/orvex/server/history.py | 2 +- packages/server/src/orvex/server/lake_open.py | 10 +- .../server/src/orvex/server/lake_refusals.py | 2 +- .../src/orvex/server/lake_separation.py | 27 +- packages/server/src/orvex/server/leftovers.py | 2 +- packages/server/src/orvex/server/lineage.py | 4 +- .../server/src/orvex/server/maintenance.py | 4 +- .../src/orvex/server/permission_state.py | 4 +- .../server/src/orvex/server/project_check.py | 7 +- .../server/src/orvex/server/query_cancel.py | 5 +- packages/server/src/orvex/server/run.py | 4 +- packages/server/src/orvex/server/run_reads.py | 4 +- packages/server/src/orvex/server/run_steps.py | 2 +- packages/server/src/orvex/server/scheduler.py | 2 +- .../server/src/orvex/server/semantic_http.py | 10 +- .../server/src/orvex/server/semantic_model.py | 4 +- .../server/src/orvex/server/store_access.py | 33 + packages/server/src/orvex/server/validate.py | 2 +- packages/store/pyproject.toml | 2 +- packages/store/src/orvex/store/__init__.py | 5 +- .../store/src/orvex/store/api/__init__.py | 6 + packages/store/src/orvex/store/api/held.py | 107 + packages/store/src/orvex/store/api/service.py | 500 +++ .../store/src/orvex/store/catalog_reads.py | 2 +- packages/store/src/orvex/store/clickhouse.py | 42 +- packages/store/src/orvex/store/credentials.py | 2 +- packages/store/src/orvex/store/errors.py | 79 - .../store/src/orvex/store/existing_lake.py | 12 +- .../src/orvex/store/existing_lake_guard.py | 2 +- .../src/orvex/store/existing_lake_reads.py | 5 +- packages/store/src/orvex/store/file_names.py | 2 +- .../store/src/orvex/store/file_sources.py | 2 +- packages/store/src/orvex/store/isolation.py | 4 +- .../store/src/orvex/store/json_encoding.py | 46 +- packages/store/src/orvex/store/lake.py | 6 +- packages/store/src/orvex/store/lake_names.py | 2 +- packages/store/src/orvex/store/limits.py | 19 +- packages/store/src/orvex/store/locations.py | 2 +- packages/store/src/orvex/store/maintenance.py | 2 +- packages/store/src/orvex/store/module.toml | 13 + .../store/src/orvex/store/nested_types.py | 2 +- packages/store/src/orvex/store/parameters.py | 2 +- packages/store/src/orvex/store/read_stream.py | 2 +- packages/store/src/orvex/store/read_tree.py | 2 +- packages/store/src/orvex/store/separation.py | 91 +- .../store/src/orvex/store/source_reader.py | 4 +- .../store/src/orvex/store/source_worker.py | 2 +- packages/store/src/orvex/store/sql_guard.py | 2 +- packages/store/src/orvex/store/sql_parser.py | 2 +- packages/store/src/orvex/store/sqlite_copy.py | 4 +- packages/store/src/orvex/store/transaction.py | 2 +- pyproject.toml | 2 + tools/benchmarks/query_baseline.py | 6 +- tools/benchmarks/server_equivalence.py | 2 +- tools/benchmarks/source_stream.py | 168 - tools/benchmarks/store_baseline.py | 16 +- tools/benchmarks/store_equivalence.py | 16 +- tools/checks/materialization-allowlist.toml | 10 + tools/ci/checks.toml | 22 + 90 files changed, 6069 insertions(+), 548 deletions(-) create mode 100644 packages/contracts/src/orvex/contracts/store/arrow_ipc.py create mode 100644 packages/contracts/src/orvex/contracts/store/client.py create mode 100644 packages/contracts/src/orvex/contracts/store/client_sources.py create mode 100644 packages/contracts/src/orvex/contracts/store/interface.py create mode 100644 packages/contracts/src/orvex/contracts/store/kit.py create mode 100644 packages/contracts/src/orvex/contracts/store/kit_support.py create mode 100644 packages/contracts/src/orvex/contracts/store/values.py create mode 100644 packages/server/src/orvex/server/store_access.py create mode 100644 packages/store/src/orvex/store/api/__init__.py create mode 100644 packages/store/src/orvex/store/api/held.py create mode 100644 packages/store/src/orvex/store/api/service.py delete mode 100644 packages/store/src/orvex/store/errors.py create mode 100644 packages/store/src/orvex/store/module.toml delete mode 100644 tools/benchmarks/source_stream.py diff --git a/.github/workflows/interfaces.yml b/.github/workflows/interfaces.yml index 373fbe4d..8adf18c2 100644 --- a/.github/workflows/interfaces.yml +++ b/.github/workflows/interfaces.yml @@ -54,3 +54,115 @@ jobs: python -m pip install --quiet -r pylock.toml - name: The contracts module alone, outside the repository run: python -m tools.ci.standalone contracts + + standalone-store: + # The store alone (#486 §5 step 2): its wheel with only the contracts wheel, in a clean + # environment outside the repository, passing its own tests and its contract's kit, with + # the same disposable services its tests need in python.yml, so none is skipped. + runs-on: ubuntu-latest + timeout-minutes: 30 + # PostgreSQL catalog for the store tests (P1-02); a disposable database, not a secret. + services: + postgres: + # postgres:18.6-trixie + image: postgres@sha256:edb0c08dc33befd0262ab3b5cf1c4f3f8cfd8b09e88c05b45adee29d6761ddb6 + env: + POSTGRES_PASSWORD: ci-disposable-database + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 2s --health-timeout 5s --health-retries 30 + # ClickHouse for the Direct Query tests (P2-31, #225); a disposable server, not a secret. + # The tests create their own read-only profiles, users and synthetic tables. + clickhouse: + # clickhouse/clickhouse-server:26.10.2.103 (#429: the latest stable) + image: clickhouse/clickhouse-server@sha256:b2bfa4d8399ee9973868cc5c1754cb506a6f4b86fbddc41010cc090beb55c85e + env: + CLICKHOUSE_USER: ci + CLICKHOUSE_PASSWORD: ci-disposable-database + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + ports: ["8123:8123"] + options: >- + --health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'" + --health-interval 2s --health-timeout 5s --health-retries 60 + env: + STORE_TEST_POSTGRES: host=localhost port=5432 dbname=postgres user=postgres + STORE_TEST_POSTGRES_PASSWORD: ci-disposable-database + STORE_TEST_CLICKHOUSE: localhost:8123:ci + STORE_TEST_CLICKHOUSE_PASSWORD: ci-disposable-database + # The PostgreSQL with TLS started below (#120), reached through 127.0.0.2: a host that is + # not the local machine for the catalog rule, so verify-full is required and tested. + STORE_TEST_POSTGRES_TLS: host=127.0.0.2 port=5433 dbname=postgres user=postgres + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Start a PostgreSQL with TLS for the catalog tests (#120) and Azurite (#127) + # Started before the Python install and waited for after it (#373): the Azurite image + # downloads while Python and the lock file install. + # A certificate authority and a server certificate made for this job only: disposable, + # never committed, not secrets. The CA file is given to the tests by its path. + run: | + dir="$RUNNER_TEMP/postgres-tls" + mkdir -p "$dir" + cd "$dir" + openssl req -x509 -newkey rsa:3072 -nodes -days 2 -subj "/CN=ci-test-ca" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" \ + -keyout ca.key -out ca.crt + openssl req -newkey rsa:3072 -nodes -subj "/CN=localhost" \ + -keyout server.key -out server.csr + printf '%s\n' "basicConstraints=CA:FALSE" \ + "keyUsage=critical,digitalSignature,keyEncipherment" \ + "extendedKeyUsage=serverAuth" \ + "subjectAltName=DNS:localhost,IP:127.0.0.1,IP:127.0.0.2" > server.ext + openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -days 2 \ + -extfile server.ext -out server.crt + rm ca.key server.csr + # The image's postgres user (uid 999) must own the key, readable by it only. + sudo chown 999:999 server.key server.crt + sudo chmod 600 server.key + # postgres:18.6-trixie, the same image as the service above. + docker run --detach --name postgres-tls --publish 5433:5432 \ + --env POSTGRES_PASSWORD=ci-disposable-database \ + --volume "$dir/server.crt:/tls/server.crt:ro" \ + --volume "$dir/server.key:/tls/server.key:ro" \ + postgres@sha256:edb0c08dc33befd0262ab3b5cf1c4f3f8cfd8b09e88c05b45adee29d6761ddb6 \ + -c ssl=on -c ssl_cert_file=/tls/server.crt -c ssl_key_file=/tls/server.key + echo "STORE_TEST_POSTGRES_TLS_CA=$dir/ca.crt" >> "$GITHUB_ENV" + # azurite 3.37.0, blob service only, downloaded and started in the background; its + # output goes to a file the wait below shows when it fails. + nohup docker run --detach --name azurite --publish 10000:10000 \ + mcr.microsoft.com/azure-storage/azurite@sha256:830430c1da1a2d537e08f3e6764dd1f5ae00cf0346bcaf625b968ec3f0971fd5 \ + azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --skipApiVersionCheck --loose \ + > "$RUNNER_TEMP/azurite-start.log" 2>&1 & + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: "3.15" + - name: Install the build tools from the lock file + run: | + python -m pip install --quiet pip==26.3 + python -m pip install --quiet -r pylock.toml + - name: Install the store's DuckDB extensions, verified (#427) + run: python -m tools.ci.duckdb_extensions + - name: Wait for the PostgreSQL with TLS and Azurite + # The PostgreSQL is ready when it accepts TCP connections (the first start listens on a + # socket only). Azurite's development account is public and disposable, not a secret. + run: | + for attempt in $(seq 60); do + if docker exec postgres-tls pg_isready -U postgres -h 127.0.0.1 -q; then break; fi + sleep 1 + done + docker exec postgres-tls pg_isready -U postgres -h 127.0.0.1 + for attempt in $(seq 120); do + if curl --silent --output /dev/null http://127.0.0.1:10000/; then break; fi + sleep 1 + done + if ! curl --silent --output /dev/null http://127.0.0.1:10000/; then + cat "$RUNNER_TEMP/azurite-start.log" + exit 1 + fi + key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + echo "STORE_TEST_AZURE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=$key;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" >> "$GITHUB_ENV" + - name: The store module alone, outside the repository + run: python -m tools.ci.standalone store diff --git a/.github/workflows/property-nightly.yml b/.github/workflows/property-nightly.yml index 678c3fd3..1d68f90a 100644 --- a/.github/workflows/property-nightly.yml +++ b/.github/workflows/property-nightly.yml @@ -34,6 +34,6 @@ jobs: env: PROPERTY_PROFILE: nightly run: | - python -m pytest -q -p tools.ci.timing --junitxml=junit.xml tests/properties + python -m pytest -q -p tools.ci.timing --junitxml=junit.xml tests/properties packages/store/tests/test_clickhouse_scanner_fuzzing.py # Each test's time to this log (#375): what test-trends.yml reads back. python -m tools.ci.durations emit junit.xml diff --git a/benchmarks/source-stream.md b/benchmarks/source-stream.md index 6658008e..2fdc1a93 100644 --- a/benchmarks/source-stream.md +++ b/benchmarks/source-stream.md @@ -1,6 +1,6 @@ # Source isolation cost (#372, ADR-0031 §6) -Measured with `tools/benchmarks/source_stream.py` on 2026-11-07. Every number below comes from that run; none is estimated. +Measured with `tools/benchmarks/source_stream.py` (since #486, `packages/store/benchmarks/source_stream.py`) on 2026-11-07. Every number below comes from that run; none is estimated. - **Code:** the #372 branch on top of `493426e` (main), with the script as committed here; the measured code is that of this pull request - **Command:** `python -m tools.benchmarks.source_stream --rows 10000000 --rows 50000000 --runs 3` diff --git a/conformance/runners/corpus_clickhouse.py b/conformance/runners/corpus_clickhouse.py index e83a280a..cd53c021 100644 --- a/conformance/runners/corpus_clickhouse.py +++ b/conformance/runners/corpus_clickhouse.py @@ -1,7 +1,7 @@ """Running a corpus case on ClickHouse (P2-31 PR 4, #225; ADR-0027 §4, ADR-0028 §3): the case's tables loaded into a ClickHouse database with the mapped types, and the case answered there through the product's own wiring: the store's verified, read-only connection -(`orvex.store.clickhouse`), the server's adapter (`clickhouse_adapter`) and the evidence its +(its contract's `clickhouse`), the server's adapter (`clickhouse_adapter`) and the evidence its schedule verifies (`clickhouse_evidence`). The server is `CORPUS_CLICKHOUSE` (`host:port:user`, an administrator of a disposable server) @@ -30,7 +30,11 @@ from pathlib import Path from typing import Any +from orvex.contracts.store.client import Lake +from orvex.contracts.store.client_sources import connect_clickhouse +from orvex.contracts.store.interface import REQUIRED_SETTINGS from orvex.contracts.store.v1 import ReadRequest, TablesRequest +from orvex.contracts.store.values import Budget, ClickHouseTarget from orvex.semantic.adapters import clickhouse as ch from orvex.semantic.adapters.clickhouse import CAPABILITIES as CLICKHOUSE_CAPABILITIES from orvex.semantic.compiler.compiled import CompiledModel @@ -42,8 +46,7 @@ from orvex.server.clickhouse_engines import tables from orvex.server.clickhouse_evidence import EvidenceSchedule, checked from orvex.server.identity import load_identity -from orvex.store.clickhouse import REQUIRED_SETTINGS, Budget, ClickHouse, ClickHouseTarget -from orvex.store.lake import Lake +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce @@ -166,7 +169,7 @@ def clickhouse( with database() as (admin, name): load(admin, name, lake) target = ClickHouseTarget(admin.host, admin.port, name, name, READER_PASSWORD, tls=False) - engine = ClickHouse.connect(target, BUDGET) + engine = connect_clickhouse(store(), target, BUDGET) entry = EngineEntry( CONNECTION, "clickhouse", diff --git a/conformance/runners/corpus_execution.py b/conformance/runners/corpus_execution.py index 9aef09a1..634991b7 100644 --- a/conformance/runners/corpus_execution.py +++ b/conformance/runners/corpus_execution.py @@ -23,12 +23,15 @@ from typing import Any, cast import yaml +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, Parameter, ParameterType, StatementRequest, ) +from orvex.contracts.store.values import Limits from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.errors import SemanticError @@ -37,9 +40,7 @@ from orvex.semantic.policies import PolicyModel, read_policies from orvex.semantic.query.binding import Limits as QueryLimits from orvex.semantic.receipt import Key -from orvex.store.errors import StoreError -from orvex.store.json_encoding import arrow_to_json -from orvex.store.lake import Lake, Limits, open_lake +from orvex.server.store_access import store from conformance.runners import bi_platform_adapter as adapter @@ -167,7 +168,7 @@ def case_lake( ) -> Iterator[Lake]: with tempfile.TemporaryDirectory() as folder: location = LocalLakeLocation(kind="lake_location", catalog="local", path=folder) - lake = open_lake(location, Limits("1GB", 2, 2, 0, 600_000)) + lake = store().open_lake(location, Limits("1GB", 2, 2, 0, 600_000)) try: load_tables(lake, inputs, catalog, schema) yield lake @@ -179,7 +180,7 @@ def engine_for(lake: Lake, **change: Any) -> Engine: # noqa: ANN401 snapshot = lake.current_snapshot() arguments: dict[str, Any] = { "read": lake.read, - "encoder": arrow_to_json, + "encoder": store().arrow_to_json, "snapshot": {"id": snapshot.id, "committed_at": snapshot.committed_at.isoformat()}, "engine_version": version("duckdb"), "result_key": RESULT_KEY, @@ -217,7 +218,7 @@ def viewer_context( def rows_of(found: Answer) -> list[list[str | None]]: """The result's values as `store/v1` JSON text, positionally.""" - document = json.loads(found.result.wire(arrow_to_json)) + document = json.loads(found.result.wire(store().arrow_to_json)) return [[None if v is None else str(v) for v in row] for row in document["rows"]] diff --git a/conformance/runners/corpus_existing_ducklake.py b/conformance/runners/corpus_existing_ducklake.py index ccae666d..0c2b5f08 100644 --- a/conformance/runners/corpus_existing_ducklake.py +++ b/conformance/runners/corpus_existing_ducklake.py @@ -18,6 +18,7 @@ from __future__ import annotations +import contextlib import datetime as dt import os import shutil @@ -31,6 +32,8 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.client import Lake +from orvex.contracts.store.client_sources import open_existing_lake from orvex.contracts.store.v1 import ( ExistingLakeAttachment, PostgresLakeLocation, @@ -38,6 +41,7 @@ Snapshot, TablesRequest, ) +from orvex.contracts.store.values import ExistingLakeLimits from orvex.semantic.compiler.compiled import CompiledModel from orvex.semantic.engines import EngineEntry, Routing from orvex.semantic.execution import answer @@ -48,9 +52,7 @@ from orvex.server.existing_lake_evidence import violated from orvex.server.existing_lake_snapshots import SnapshotSchedule from orvex.server.identity import load_identity -from orvex.store.existing_lake import ExistingLake -from orvex.store.existing_lake_reads import LakeReads -from orvex.store.lake import Lake +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce @@ -214,8 +216,8 @@ def existing_ducklake( EXISTING_DUCKLAKE_CAPABILITIES, ) routing = Routing({str(d["name"]): entry for d in datasets}) - with ExistingLake(made.attachment(), memory_limit="1GB", threads=2) as existing: - reads = LakeReads(existing, timeout_ms=TIMEOUT_MS) + limits = ExistingLakeLimits(memory_limit="1GB", threads=2, timeout_ms=TIMEOUT_MS) + with contextlib.closing(open_existing_lake(store(), made.attachment(), limits)) as reads: def evidence(snapshot: Snapshot) -> str | None: return violated( diff --git a/conformance/runners/corpus_http.py b/conformance/runners/corpus_http.py index c3d851f3..dba6f131 100644 --- a/conformance/runners/corpus_http.py +++ b/conformance/runners/corpus_http.py @@ -18,15 +18,15 @@ import pyarrow as pa from fastapi.testclient import TestClient +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.compiled import CompiledModel from orvex.semantic.errors import SemanticError from orvex.semantic.policies import PolicyModel from orvex.server import semantic_http, semantic_model from orvex.server.app import LocalSemantic, create_app from orvex.server.local_access import new_access +from orvex.server.store_access import store from orvex.server.versions import Versions -from orvex.store.json_encoding import arrow_to_json -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce @@ -58,7 +58,7 @@ def model(self, lake: Lake, snapshot: int) -> semantic_model.Loaded: def _cells(envelope: Mapping[str, Any], rows: list[list[Any]], table: pa.Table) -> None: text = "".join( - arrow_to_json(pa.RecordBatchReader.from_batches(table.schema, table.to_batches())) + store().arrow_to_json(pa.RecordBatchReader.from_batches(table.schema, table.to_batches())) ) values = json.loads(text)["rows"] states = {(s["row"], s["column"]) for s in envelope.get("cell_states", [])} diff --git a/conformance/runners/security_engines.py b/conformance/runners/security_engines.py index db8025dc..1c6e9c1f 100644 --- a/conformance/runners/security_engines.py +++ b/conformance/runners/security_engines.py @@ -33,9 +33,18 @@ from typing import Any, NoReturn import duckdb +from orvex.contracts.store.client import Lake +from orvex.contracts.store.client_sources import ( + ClickHouse, + ExistingLakeReads, + connect_clickhouse, + open_existing_lake, +) +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( StatementRequest, ) +from orvex.contracts.store.values import Budget, ClickHouseTarget, ExistingLakeLimits from orvex.semantic.adapters import clickhouse as ch from orvex.semantic.adapters.clickhouse import CAPABILITIES as CLICKHOUSE_CAPABILITIES from orvex.semantic.catalog.building import catalog_for_context @@ -52,12 +61,7 @@ from orvex.server.engine_catalog import EXISTING_DUCKLAKE_CAPABILITIES from orvex.server.existing_lake_adapter import ExistingLakeAdapter from orvex.server.existing_lake_snapshots import SnapshotSchedule -from orvex.store.clickhouse import Budget, ClickHouse, ClickHouseTarget -from orvex.store.errors import StoreError -from orvex.store.existing_lake import ExistingLake -from orvex.store.existing_lake_reads import LakeReads -from orvex.store.json_encoding import arrow_to_json -from orvex.store.lake import Lake +from orvex.server.store_access import store from conformance.runners import corpus_execution as ce from conformance.runners import security_existing_lake as sx @@ -69,6 +73,7 @@ NOW = dt.datetime(2026, 11, 8, 12, 0, tzinfo=dt.UTC) # the schedules' first check READER_PASSWORD = "SECURITY_CASES_CLICKHOUSE_READER" # set for each served database only BUDGET = Budget(timeout_ms=30_000, memory_bytes=1_000_000_000) +LIMITS = ExistingLakeLimits(memory_limit="512MB", threads=1, timeout_ms=60_000) PROFILE = { "readonly": "1", "join_use_nulls": "1", @@ -174,7 +179,9 @@ def _datasets(found: sf.Fixture) -> list[str]: class _Undeclared(ExistingLakeAdapter): """A connection whose declaration was refused: it serves nothing, and says why.""" - def __init__(self, reads: LakeReads, schedule: SnapshotSchedule, refusal: StoreError) -> None: + def __init__( + self, reads: ExistingLakeReads, schedule: SnapshotSchedule, refusal: StoreError + ) -> None: super().__init__(reads, schedule, "test") self._refusal = refusal @@ -186,31 +193,25 @@ def read(self, *_: object) -> Any: # noqa: ANN401 - never reached: nothing is p def _declared( - host: sx.Host, reads: LakeReads, schedule: SnapshotSchedule, made: list[object] + host: sx.Host, reads: ExistingLakeReads, schedule: SnapshotSchedule, made: list[object] ) -> _Undeclared: """The connection declared again with its location as changed, as the server opens it: it must be refused before DuckLake attaches anything, with no read made (DQ-018).""" - attached: list[str] = [] - real = ExistingLake._attach - - def spied(lake: ExistingLake, target: str, options: list[str]) -> None: - attached.append(target) - real(lake, target, options) - + client = store() + held = client.call("held").existing_lakes reads_before = len(made) - ExistingLake._attach = spied # type: ignore[method-assign, assignment] try: - ExistingLake(host.attachment(), memory_limit="512MB", threads=1).close() + open_existing_lake(client, host.attachment(), LIMITS).close() except StoreError as error: refusal = error else: raise RuntimeError("the lake was declared with a location overlapping a protected one") - finally: - ExistingLake._attach = real # type: ignore[method-assign] if "overlaps" not in str(refusal.details.get("reason", "")): # refused, but for the overlap raise RuntimeError(f"the declaration was refused for another reason: {refusal.code}") - if attached or len(made) != reads_before: - raise RuntimeError("the lake was attached or read before its declaration was refused") + # That nothing is attached before the refusal is the store's own test + # (`test_existing_lake_files.py`); here, nothing is held and nothing is read. + if client.call("held").existing_lakes != held or len(made) != reads_before: + raise RuntimeError("the lake was held or read before its declaration was refused") return _Undeclared(reads, schedule, refusal) @@ -245,9 +246,8 @@ def served( EXISTING_DUCKLAKE_CAPABILITIES, ) with sx.existing_lake(found, catalog, unavailable) as host: - existing = ExistingLake(host.attachment(), memory_limit="512MB", threads=1) + reads = open_existing_lake(store(), host.attachment(), LIMITS) try: - reads = LakeReads(existing, timeout_ms=60_000) made: list[object] = [] reading = reads.read @@ -299,7 +299,7 @@ def lake_changed(step: Mapping[str, Any]) -> None: lake_change=lake_changed, ) finally: - existing.close() + reads.close() return with _clickhouse(found, clickhouse or default_clickhouse()) as (server, client): routing, adapters = _wired(client, found.compiled) @@ -356,7 +356,7 @@ def _clickhouse( _load(admin, table, found.catalog[table], found.rows[table]) os.environ[READER_PASSWORD] = password target = ClickHouseTarget(admin.host, admin.port, name, name, READER_PASSWORD, tls=False) - yield admin, ClickHouse.connect(target, BUDGET) + yield admin, connect_clickhouse(store(), target, BUDGET) finally: os.environ.pop(READER_PASSWORD, None) admin.database = "" @@ -449,7 +449,7 @@ def ask( [column_name(c) for c in envelope["columns"]], [str(n["code"]) for n in envelope.get("notices", [])], None, - [got.result.wire(arrow_to_json), json.dumps(view, default=str)], + [got.result.wire(store().arrow_to_json), json.dumps(view, default=str)], ) return (received, got) if whole else received diff --git a/conformance/runners/security_fixtures.py b/conformance/runners/security_fixtures.py index a05c4118..83995d48 100644 --- a/conformance/runners/security_fixtures.py +++ b/conformance/runners/security_fixtures.py @@ -22,11 +22,11 @@ from pathlib import Path from typing import Any +from orvex.contracts.store.client import Lake from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model from orvex.semantic.ossie.reader import read from orvex.semantic.policies import PolicyModel, read_policies -from orvex.store.lake import Lake from conformance.runners import corpus_execution as ce diff --git a/conformance/runners/security_http.py b/conformance/runners/security_http.py index ad1d8d6d..611a7e07 100644 --- a/conformance/runners/security_http.py +++ b/conformance/runners/security_http.py @@ -30,11 +30,11 @@ from orvex.access.directory import Bootstrap, Directory from orvex.access.directory.attributes import Attributes from orvex.access.storage import SqliteStore +from orvex.contracts.store.client import Lake from orvex.server import semantic_http, semantic_model from orvex.server.identity import load_identity from orvex.server.server_app import ServedProject, create_server_app from orvex.server.versions import Versions -from orvex.store.lake import Lake from conformance.runners import security_fixtures as sf from conformance.runners.security_engines import Received, column_name, managed_delete diff --git a/packages/contracts/interface-record.json b/packages/contracts/interface-record.json index 50cf3182..13ba6cae 100644 --- a/packages/contracts/interface-record.json +++ b/packages/contracts/interface-record.json @@ -1,3 +1,2815 @@ { - "contracts": {} + "contracts": { + "store": { + "capabilities": [ + "clickhouse", + "encoding", + "existing-lake", + "lake", + "maintenance", + "read", + "separation", + "write" + ], + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "kit": { + "cases": { + "begin.idempotent": { + "expectation": "one key begins one transaction", + "idempotency": true, + "operation": "begin", + "since": "1.0" + }, + "cancel-read.idempotent": { + "expectation": "one key cancels once", + "idempotency": true, + "operation": "cancel_read", + "since": "1.0" + }, + "cancel-transaction.idempotent": { + "expectation": "one key cancels once", + "idempotency": true, + "operation": "cancel_transaction", + "since": "1.0" + }, + "cancel-transaction.rolls-back": { + "expectation": "a cancelled transaction never commits", + "idempotency": false, + "operation": "cancel_transaction", + "since": "1.0" + }, + "changes.removed": { + "expectation": "removed rows are counted; `since` after `snapshot` refused", + "idempotency": false, + "operation": "changes", + "since": "1.0" + }, + "check-apart.protected": { + "expectation": "a lake inside a protected location is refused", + "idempotency": false, + "operation": "check_apart", + "since": "1.0" + }, + "clickhouse-catalog.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "clickhouse_catalog", + "since": "1.0" + }, + "clickhouse-columns.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "clickhouse_columns", + "since": "1.0" + }, + "clickhouse-estimate.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "clickhouse_estimate", + "since": "1.0" + }, + "clickhouse-now.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "clickhouse_now", + "since": "1.0" + }, + "close-clickhouse.harmless": { + "expectation": "closing what is not held changes nothing", + "idempotency": false, + "operation": "close_clickhouse", + "since": "1.0" + }, + "close-existing-lake.idempotent": { + "expectation": "one key detaches once", + "idempotency": true, + "operation": "close_existing_lake", + "since": "1.0" + }, + "close-lake.idempotent": { + "expectation": "one key closes once", + "idempotency": true, + "operation": "close_lake", + "since": "1.0" + }, + "close-read.idempotent": { + "expectation": "one key lets a read go once", + "idempotency": true, + "operation": "close_read", + "since": "1.0" + }, + "commit.guarded": { + "expectation": "a view is refused; a forged transaction is refused", + "idempotency": false, + "operation": "commit", + "since": "1.0" + }, + "commit.idempotent": { + "expectation": "one key commits once and answers the same", + "idempotency": true, + "operation": "commit", + "since": "1.0" + }, + "connect-clickhouse.guarded": { + "expectation": "plain HTTP to another machine and a missing secret are refused", + "idempotency": false, + "operation": "connect_clickhouse", + "since": "1.0" + }, + "current-snapshot.after-commit": { + "expectation": "the snapshot a commit made is current", + "idempotency": false, + "operation": "current_snapshot", + "since": "1.0" + }, + "current-snapshot.forged-handle": { + "expectation": "a forged lake handle is refused", + "idempotency": false, + "operation": "current_snapshot", + "since": "1.0" + }, + "describe-source.landed-types": { + "expectation": "a source's columns as they land; an unknown source refused", + "idempotency": false, + "operation": "describe_source", + "since": "1.0" + }, + "describe.binder-only": { + "expectation": "a SELECT's columns from the binder; anything else refused", + "idempotency": false, + "operation": "describe", + "since": "1.0" + }, + "execute.guarded": { + "expectation": "ATTACH, COPY TO and SET are refused in a transaction", + "idempotency": false, + "operation": "execute", + "since": "1.0" + }, + "execute.idempotent": { + "expectation": "one key inserts once", + "idempotency": true, + "operation": "execute", + "since": "1.0" + }, + "existing-committed-at.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "existing_committed_at", + "since": "1.0" + }, + "existing-latest.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "existing_latest", + "since": "1.0" + }, + "existing-latest.reads": { + "expectation": "latest, verify, committed-at, columns, views and a read, exact", + "idempotency": false, + "operation": "existing_latest", + "since": "1.0" + }, + "existing-table-columns.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "existing_table_columns", + "since": "1.0" + }, + "existing-verify.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "existing_verify", + "since": "1.0" + }, + "existing-views.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "existing_views", + "since": "1.0" + }, + "held.lists-open": { + "expectation": "what is held is listed; a closed lake is let go and refused", + "idempotency": false, + "operation": "held", + "since": "1.0" + }, + "is-object-storage.schemes": { + "expectation": "s3://, gs:// and az:// only, case as written", + "idempotency": false, + "operation": "is_object_storage", + "since": "1.0" + }, + "json-frame.columns": { + "expectation": "the columns with their DuckDB types and the closing tail; no type is refused", + "idempotency": false, + "operation": "json_frame", + "since": "1.0" + }, + "json-rows.exact": { + "expectation": "exact text, NULL as null, rows separated", + "idempotency": false, + "operation": "json_rows", + "since": "1.0" + }, + "land.idempotent": { + "expectation": "one key lands once", + "idempotency": true, + "operation": "land", + "since": "1.0" + }, + "load-arrow.exact": { + "expectation": "a DECIMAL and a NULL are loaded exactly", + "idempotency": false, + "operation": "load_arrow", + "since": "1.0" + }, + "load-arrow.idempotent": { + "expectation": "one key loads once", + "idempotency": true, + "operation": "load_arrow", + "since": "1.0" + }, + "location-components.parts": { + "expectation": "object storage and local paths by parts; anything else refused", + "idempotency": false, + "operation": "location_components", + "since": "1.0" + }, + "locations-overlap.prefix": { + "expectation": "inside overlaps, a sibling with the same prefix does not", + "idempotency": false, + "operation": "locations_overlap", + "since": "1.0" + }, + "maintain.idempotent": { + "expectation": "one key maintains once and answers the same", + "idempotency": true, + "operation": "maintain", + "since": "1.0" + }, + "maintain.naive-cutoff": { + "expectation": "a cutoff without a zone is refused", + "idempotency": false, + "operation": "maintain", + "since": "1.0" + }, + "open-existing-lake.guarded": { + "expectation": "encryption and a missing protected set are refused", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.idempotent": { + "expectation": "one key attaches once", + "idempotency": true, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-lake.idempotent": { + "expectation": "one key opens one lake", + "idempotency": true, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.limits-required": { + "expectation": "a lake without every limit is refused (law 7)", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.source-not-found": { + "expectation": "a missing source file is refused before anything opens", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "read-batch.ends": { + "expectation": "every row once, then None; a forged read is refused", + "idempotency": false, + "operation": "read_batch", + "since": "1.0" + }, + "rollback.discards": { + "expectation": "a rolled-back transaction writes nothing and is let go", + "idempotency": false, + "operation": "rollback", + "since": "1.0" + }, + "rollback.idempotent": { + "expectation": "one key rolls back once", + "idempotency": true, + "operation": "rollback", + "since": "1.0" + }, + "snapshot.kept-or-none": { + "expectation": "a kept snapshot is found; an unknown or negative id is None", + "idempotency": false, + "operation": "snapshot", + "since": "1.0" + }, + "start-clickhouse-read.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "start_clickhouse_read", + "since": "1.0" + }, + "start-existing-read.forged-handle": { + "expectation": "a forged handle is refused", + "idempotency": false, + "operation": "start_existing_read", + "since": "1.0" + }, + "start-read.exact": { + "expectation": "DECIMAL, HUGEINT, TIMESTAMPTZ and NULL are read exactly", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "start-read.guarded": { + "expectation": "file readers, writes, two statements and unknown snapshots are refused", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "start-read.parameters": { + "expectation": "a parameter is bound by name with its type", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "statistics.counts": { + "expectation": "every row is counted; an unknown table refused", + "idempotency": false, + "operation": "statistics", + "since": "1.0" + }, + "table-columns.as-of": { + "expectation": "the columns as of the snapshot asked", + "idempotency": false, + "operation": "table_columns", + "since": "1.0" + }, + "tables.lower-cased": { + "expectation": "every table, lower-cased, in order", + "idempotency": false, + "operation": "tables", + "since": "1.0" + }, + "value.first-cell": { + "expectation": "the first cell as text; None when there is no row", + "idempotency": false, + "operation": "value", + "since": "1.0" + } + }, + "recorded": 58 + }, + "name": "store", + "operations": { + "begin": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + } + ], + "returns": "orvex.contracts.store.values.TransactionHandle", + "since": "1.0" + }, + "cancel_read": { + "capability": "read", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "read", + "type": "orvex.contracts.store.values.ReadHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "cancel_transaction": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "changes": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.ChangesRequest" + } + ], + "returns": "orvex.contracts.store.v1.ChangesResult", + "since": "1.0" + }, + "check_apart": { + "capability": "separation", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "name", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "data_path", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "catalog", + "type": "str | None" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "protected", + "type": "tuple[str, ...]" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "protected_files", + "type": "tuple[str, ...]" + } + ], + "returns": "None", + "since": "1.0" + }, + "clickhouse_catalog": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "engine", + "type": "orvex.contracts.store.values.ClickHouseHandle" + } + ], + "returns": "orvex.contracts.store.values.ClickHouseCatalog", + "since": "1.0" + }, + "clickhouse_columns": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "engine", + "type": "orvex.contracts.store.values.ClickHouseHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "table", + "type": "str" + } + ], + "returns": "tuple[orvex.contracts.store.values.SourceColumn, ...]", + "since": "1.0" + }, + "clickhouse_estimate": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "engine", + "type": "orvex.contracts.store.values.ClickHouseHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "sql", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "parameters", + "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" + } + ], + "returns": "tuple[dict[str, str | int | float | bool | None], ...]", + "since": "1.0" + }, + "clickhouse_now": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "engine", + "type": "orvex.contracts.store.values.ClickHouseHandle" + } + ], + "returns": "datetime.datetime", + "since": "1.0" + }, + "close_clickhouse": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "engine", + "type": "orvex.contracts.store.values.ClickHouseHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "close_existing_lake": { + "capability": "existing-lake", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "close_lake": { + "capability": "lake", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "close_read": { + "capability": "read", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "read", + "type": "orvex.contracts.store.values.ReadHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "commit": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + } + ], + "returns": "orvex.contracts.store.v1.CommitResult", + "since": "1.0" + }, + "connect_clickhouse": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "target", + "type": "orvex.contracts.store.values.ClickHouseTarget" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "budget", + "type": "orvex.contracts.store.values.Budget" + } + ], + "returns": "orvex.contracts.store.values.ClickHouseHandle", + "since": "1.0" + }, + "current_snapshot": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + } + ], + "returns": "orvex.contracts.store.v1.Snapshot", + "since": "1.0" + }, + "describe": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "sql", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "parameters", + "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" + } + ], + "returns": "tuple[tuple[str, str], ...]", + "since": "1.0" + }, + "describe_source": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "name", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "sql", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "parameters", + "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" + } + ], + "returns": "tuple[tuple[str, str], ...]", + "since": "1.0" + }, + "execute": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.StatementRequest" + } + ], + "returns": "None", + "since": "1.0" + }, + "existing_committed_at": { + "capability": "existing-lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "snapshot", + "type": "int" + } + ], + "returns": "datetime.datetime | None", + "since": "1.0" + }, + "existing_latest": { + "capability": "existing-lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + } + ], + "returns": "orvex.contracts.store.v1.Snapshot", + "since": "1.0" + }, + "existing_table_columns": { + "capability": "existing-lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "snapshot", + "type": "orvex.contracts.store.v1.Snapshot" + } + ], + "returns": "orvex.contracts.store.v1.TablesResult", + "since": "1.0" + }, + "existing_verify": { + "capability": "existing-lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "snapshot", + "type": "orvex.contracts.store.v1.Snapshot" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "current", + "type": "orvex.contracts.store.v1.ExistingLakeAttachment" + } + ], + "returns": "orvex.contracts.store.v1.Snapshot", + "since": "1.0" + }, + "existing_views": { + "capability": "existing-lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "snapshot", + "type": "orvex.contracts.store.v1.Snapshot" + } + ], + "returns": "tuple[str, ...]", + "since": "1.0" + }, + "held": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + } + ], + "returns": "orvex.contracts.store.values.Held", + "since": "1.0" + }, + "is_object_storage": { + "capability": "separation", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "location", + "type": "str" + } + ], + "returns": "bool", + "since": "1.0" + }, + "json_frame": { + "capability": "encoding", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "schema", + "type": "bytes" + } + ], + "returns": "orvex.contracts.store.values.JsonFrame", + "since": "1.0" + }, + "json_rows": { + "capability": "encoding", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "batch", + "type": "bytes" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "first", + "type": "bool" + } + ], + "returns": "tuple[str, ...]", + "since": "1.0" + }, + "land": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "source", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "sql", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "parameters", + "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "schema", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "table", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "timeout_ms", + "type": "int | None" + } + ], + "returns": "int", + "since": "1.0" + }, + "load_arrow": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "schema", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "table", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "data", + "type": "bytes" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "columns", + "type": "tuple[tuple[str, str], ...]" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "timeout_ms", + "type": "int | None" + } + ], + "returns": "int", + "since": "1.0" + }, + "location_components": { + "capability": "separation", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "name", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "location", + "type": "str" + } + ], + "returns": "tuple[str, ...]", + "since": "1.0" + }, + "locations_overlap": { + "capability": "separation", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "name", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "first", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "second", + "type": "str" + } + ], + "returns": "bool", + "since": "1.0" + }, + "maintain": { + "capability": "maintenance", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "expire_before", + "type": "datetime.datetime" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "orphans_before", + "type": "datetime.datetime" + } + ], + "returns": "orvex.contracts.store.v1.MaintenanceReport", + "since": "1.0" + }, + "open_existing_lake": { + "capability": "existing-lake", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "attachment", + "type": "orvex.contracts.store.v1.ExistingLakeAttachment" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "limits", + "type": "orvex.contracts.store.values.ExistingLakeLimits" + } + ], + "returns": "orvex.contracts.store.values.ExistingLakeHandle", + "since": "1.0" + }, + "open_lake": { + "capability": "lake", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "location", + "type": "orvex.contracts.store.v1.LocalLakeLocation | orvex.contracts.store.v1.SqliteLakeLocation | orvex.contracts.store.v1.PostgresLakeLocation" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "limits", + "type": "orvex.contracts.store.values.Limits" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "sources", + "type": "tuple[orvex.contracts.store.v1.FileSourceAttachment | orvex.contracts.store.v1.SqliteSourceAttachment | orvex.contracts.store.v1.PostgresSourceAttachment, ...]" + } + ], + "returns": "orvex.contracts.store.values.LakeHandle", + "since": "1.0" + }, + "read_batch": { + "capability": "read", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "read", + "type": "orvex.contracts.store.values.ReadHandle" + } + ], + "returns": "bytes | None", + "since": "1.0" + }, + "rollback": { + "capability": "write", + "changes_state": true, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + } + ], + "returns": "None", + "since": "1.0" + }, + "snapshot": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "snapshot_id", + "type": "int" + } + ], + "returns": "orvex.contracts.store.v1.Snapshot | None", + "since": "1.0" + }, + "start_clickhouse_read": { + "capability": "clickhouse", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "engine", + "type": "orvex.contracts.store.values.ClickHouseHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "sql", + "type": "str" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "parameters", + "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "columns", + "type": "tuple[tuple[str, str], ...]" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "timeout_ms", + "type": "int | None" + } + ], + "returns": "orvex.contracts.store.values.ReadHandle", + "since": "1.0" + }, + "start_existing_read": { + "capability": "existing-lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.ExistingLakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.ReadRequest" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "committed_at", + "type": "datetime.datetime" + } + ], + "returns": "orvex.contracts.store.values.ReadHandle", + "since": "1.0" + }, + "start_read": { + "capability": "read", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.ReadRequest" + } + ], + "returns": "orvex.contracts.store.values.ReadHandle", + "since": "1.0" + }, + "statistics": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.StatisticsRequest" + } + ], + "returns": "orvex.contracts.store.v1.StatisticsResult", + "since": "1.0" + }, + "table_columns": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.TablesRequest" + } + ], + "returns": "orvex.contracts.store.v1.TablesResult", + "since": "1.0" + }, + "tables": { + "capability": "lake", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "lake", + "type": "orvex.contracts.store.values.LakeHandle" + } + ], + "returns": "tuple[tuple[str, str], ...]", + "since": "1.0" + }, + "value": { + "capability": "write", + "changes_state": false, + "errors": [ + "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", + "SOURCE_NOT_FOUND", + "SOURCE_NOT_READ_ONLY", + "STORE_BUSY", + "STORE_CANCELLED", + "STORE_CATALOG_UNAVAILABLE", + "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_PARAMETER_INVALID", + "STORE_QUERY_FAILED", + "STORE_SECRET_INLINE", + "STORE_SECRET_MISSING", + "STORE_SNAPSHOT_EXPIRED", + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SOURCE_FAILED", + "STORE_SOURCE_INVALID", + "STORE_STATEMENT_INVALID", + "STORE_TIMEOUT", + "STORE_TLS_REQUIRED", + "STORE_TRANSACTION_CONFLICT", + "STORE_TYPE_UNSUPPORTED", + "UNSUPPORTED_SOURCE_TYPE" + ], + "parameters": [ + { + "default": false, + "kind": "positional-or-keyword", + "name": "envelope", + "type": "orvex.contracts.envelope.CallEnvelope" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "transaction", + "type": "orvex.contracts.store.values.TransactionHandle" + }, + { + "default": false, + "kind": "positional-or-keyword", + "name": "request", + "type": "orvex.contracts.store.v1.StatementRequest" + } + ], + "returns": "str | None", + "since": "1.0" + } + }, + "version": "1.0" + } + } } diff --git a/packages/contracts/src/orvex/contracts/store/arrow_ipc.py b/packages/contracts/src/orvex/contracts/store/arrow_ipc.py new file mode 100644 index 00000000..d0491099 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/arrow_ipc.py @@ -0,0 +1,37 @@ +"""How Arrow crosses the store's interface contract, on both sides: as IPC streams, exactly +(every type, its metadata and every NULL as written), never as a live reader.""" + +from __future__ import annotations + +from collections.abc import Iterable + +import pyarrow as pa + + +def stream_bytes(schema: pa.Schema, batches: Iterable[pa.RecordBatch] = ()) -> bytes: + """`batches` as one IPC stream of `schema` (dictionaries included); no batch: the schema.""" + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, schema) as writer: + for batch in batches: + writer.write_batch(batch) + written: bytes = sink.getvalue().to_pybytes() + return written + + +def batch_bytes(batch: pa.RecordBatch, schema: pa.Schema | None = None) -> bytes: + """One batch as an IPC stream of its own, written with `schema` (its own by default).""" + return stream_bytes(batch.schema if schema is None else schema, (batch,)) + + +def stream_reader(data: bytes) -> pa.RecordBatchReader: + """The reader of one IPC stream: its batches are decoded as they are read.""" + return pa.ipc.open_stream(pa.py_buffer(data)) + + +def stream_schema(data: bytes) -> pa.Schema: + return stream_reader(data).schema + + +def one_batch(data: bytes) -> pa.RecordBatch: + """The one batch of a stream written by `batch_bytes`.""" + return stream_reader(data).read_next_batch() diff --git a/packages/contracts/src/orvex/contracts/store/client.py b/packages/contracts/src/orvex/contracts/store/client.py new file mode 100644 index 00000000..558554aa --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/client.py @@ -0,0 +1,316 @@ +"""The consumer's side of the store's interface contract: `StoreClient` and the lake, transaction +and read it opens, every call sent through a binding with its call envelope (#486 §2 item 6). + +A consumer is given a binding of the chosen implementation (`orvex.contracts.binding.Bound`) and +the caller it acts for; it never knows the module behind it. Arrow arrives as IPC streams and is +handed on as a `pyarrow.RecordBatchReader`; a read the consumer drops or finishes is let go at +the store, so it never keeps an admission slot. +""" + +from __future__ import annotations + +import contextlib +import datetime as dt +import uuid +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from typing import Any + +import pyarrow as pa +from orvex.contracts.binding import Bound +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch, stream_bytes, stream_schema +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.interface import STORE, Location, Source +from orvex.contracts.store.v1 import ( + ChangesRequest, + ChangesResult, + CommitResult, + LakeLocation, + MaintenanceReport, + Parameter, + ReadRequest, + Snapshot, + SourceAttachment, + StatementRequest, + StatisticsRequest, + StatisticsResult, + TablesRequest, + TablesResult, +) +from orvex.contracts.store.values import JsonFrame, LakeHandle, Limits, TransactionHandle +from orvex.contracts.store.values import ReadHandle as ReadTicket + + +@dataclass(frozen=True, slots=True) +class Caller: + """Who a consumer's calls act for: the envelope fields besides the contract's own.""" + + actor: str + project_id: str + revision: str + correlation_id: str + + +class StoreClient: + """The store, reached through `bound` for `caller`.""" + + __slots__ = ("_bound", "_caller") + + def __init__(self, bound: Bound, caller: Caller) -> None: + if bound.contract is not STORE: + raise ValueError(f"a store client needs the store contract, not {bound.contract.name}") + self._bound = bound + self._caller = caller + + @property + def caller(self) -> Caller: + return self._caller + + def acting_for(self, caller: Caller) -> StoreClient: + """The same store, for another caller (a new correlation id included).""" + return StoreClient(self._bound, caller) + + def call(self, operation: str, /, *arguments: object) -> Any: # noqa: ANN401 - its result + changes = STORE.operation(operation).changes_state + envelope = CallEnvelope( + contract_name=STORE.name, + contract_version=str(STORE.version), + correlation_id=self._caller.correlation_id, + actor=self._caller.actor, + project_id=self._caller.project_id, + revision=self._caller.revision, + idempotency_key=uuid.uuid4().hex if changes else None, + ) + return self._bound.call(operation, envelope, *arguments) + + def open_lake( + self, + location: LakeLocation | Location, + limits: Limits, + sources: Sequence[SourceAttachment | Source] = (), + ) -> Lake: + where = location.root if isinstance(location, LakeLocation) else location + attached = tuple(s.root if isinstance(s, SourceAttachment) else s for s in sources) + return Lake(self, self.call("open_lake", where, limits, attached)) + + def reader(self, ticket: ReadTicket) -> pa.RecordBatchReader: + """The Arrow stream of a running read; the read is let go when it ends or is dropped.""" + schema = stream_schema(ticket.schema) + return pa.RecordBatchReader.from_batches(schema, _Batches(self, ticket)) + + def arrow_to_json(self, reader: pa.RecordBatchReader) -> Iterator[str]: + """The `ResultJson` document of `reader` as text fragments, one batch at a time; + unsupported types are reported before any row is produced.""" + frame: JsonFrame = self.call("json_frame", stream_bytes(reader.schema)) + yield from frame.head + first = True + for batch in reader: + fragments = self.call("json_rows", batch_bytes(batch, reader.schema), first) + first = first and not fragments + yield from fragments + yield frame.tail + + def is_object_storage(self, location: str) -> bool: + return bool(self.call("is_object_storage", location)) + + def location_components(self, name: str, location: str) -> tuple[str, ...]: + """The parts two locations are compared by: `object` then the bucket and each key + segment, or `local` then the resolved path's parts; anything else is refused.""" + found: tuple[str, ...] = self.call("location_components", name, location) + return found + + def locations_overlap(self, name: str, first: str, second: str) -> bool: + return bool(self.call("locations_overlap", name, first, second)) + + def check_apart( + self, + name: str, + data_path: str, + catalog: str | None, + protected: Sequence[str], + protected_files: Sequence[str] = (), + ) -> None: + self.call("check_apart", name, data_path, catalog, tuple(protected), tuple(protected_files)) + + +class _Batches: + """A read's batches, fetched one at a time; the read is let go once, at its end, on an + error or when nothing refers to it any more.""" + + def __init__(self, client: StoreClient, ticket: ReadTicket) -> None: + self._client = client + self._ticket = ticket + self._open = True + + def __iter__(self) -> _Batches: + return self + + def __next__(self) -> pa.RecordBatch: + if not self._open: + raise StopIteration + try: + data = self._client.call("read_batch", self._ticket) + except BaseException: + self.close() + raise + if data is None: + self.close() + raise StopIteration + return one_batch(data) + + def close(self) -> None: + if self._open: + self._open = False + self._client.call("close_read", self._ticket) + + def __del__(self) -> None: + with contextlib.suppress(Exception): + self.close() + + +@dataclass(slots=True) +class ReadHandle: + """A running read: its Arrow stream and a way to cancel it from another thread.""" + + reader: pa.RecordBatchReader + ticket: ReadTicket + client: StoreClient + + def cancel(self) -> None: + self.client.call("cancel_read", self.ticket) + + +class Lake: + """One open lake. Reads may run concurrently; write transactions run one at a time.""" + + def __init__(self, client: StoreClient, handle: LakeHandle) -> None: + self._client = client + self.handle = handle + + def close(self) -> None: + self._client.call("close_lake", self.handle) + + def current_snapshot(self) -> Snapshot: + found: Snapshot = self._client.call("current_snapshot", self.handle) + return found + + def snapshot(self, snapshot_id: int) -> Snapshot | None: + """The snapshot while the lake keeps it; None once it expired or if it never existed.""" + found: Snapshot | None = self._client.call("snapshot", self.handle, snapshot_id) + return found + + def tables(self) -> set[tuple[str, str]]: + """(schema, table) of every table of the lake now, lower-cased.""" + return set(self._client.call("tables", self.handle)) + + def describe(self, sql: str, parameters: Sequence[Parameter] = ()) -> list[tuple[str, str]]: + """(name, DuckDB type) of each output column of one SELECT, from the binder only.""" + return list(self._client.call("describe", self.handle, sql, tuple(parameters))) + + def describe_source( + self, name: str, sql: str, parameters: Sequence[Parameter] = () + ) -> list[tuple[str, str]]: + """The columns and the types they land with of `sql` on the database source `name`.""" + found = self._client.call("describe_source", self.handle, name, sql, tuple(parameters)) + return list(found) + + def statistics(self, request: StatisticsRequest) -> StatisticsResult: + found: StatisticsResult = self._client.call("statistics", self.handle, request) + return found + + def changes(self, request: ChangesRequest) -> ChangesResult: + found: ChangesResult = self._client.call("changes", self.handle, request) + return found + + def table_columns(self, request: TablesRequest) -> TablesResult: + found: TablesResult = self._client.call("table_columns", self.handle, request) + return found + + def read(self, request: ReadRequest) -> pa.RecordBatchReader: + """Run one SELECT as of `request.snapshot` and return it once, as Arrow batches.""" + return self.start_read(request).reader + + def start_read(self, request: ReadRequest) -> ReadHandle: + """Like `read`, with a handle that can cancel the read from another thread.""" + ticket: ReadTicket = self._client.call("start_read", self.handle, request) + return ReadHandle(self._client.reader(ticket), ticket, self._client) + + @contextlib.contextmanager + def transaction(self) -> Iterator[Transaction]: + """Statements inside the block commit together to one snapshot, or roll back on error.""" + handle: TransactionHandle = self._client.call("begin", self.handle) + transaction = Transaction(self._client, handle) + try: + yield transaction + except BaseException: + # The block's own error is the one raised; the rollback's could only hide it. + with contextlib.suppress(StoreError): + self._client.call("rollback", handle) + raise + transaction.result = self._client.call("commit", handle) + + def maintain( + self, expire_before: dt.datetime, orphans_before: dt.datetime + ) -> MaintenanceReport: + """Expire, merge, rewrite and clean up the lake's files (`orvex.store` README).""" + found: MaintenanceReport = self._client.call( + "maintain", self.handle, expire_before, orphans_before + ) + return found + + +class Transaction: + """A write transaction; `result` is set when the block commits.""" + + def __init__(self, client: StoreClient, handle: TransactionHandle) -> None: + self._client = client + self.handle = handle + self.result: CommitResult | None = None + + @property + def started(self) -> Snapshot | None: + """The snapshot the transaction reads (#255).""" + return self.handle.started + + def execute(self, request: StatementRequest) -> None: + self._client.call("execute", self.handle, request) + + def value(self, request: StatementRequest) -> str | None: + """The first column of the first row of one SELECT inside this transaction, as text.""" + found: str | None = self._client.call("value", self.handle, request) + return found + + def land( + self, + source: str, + sql: str, + parameters: Sequence[Parameter], + schema: str, + table: str, + timeout_ms: int | None = None, + ) -> int: + """Replace `schema.table` with what `sql` returns on the database source `source`.""" + found: int = self._client.call( + "land", self.handle, source, sql, tuple(parameters), schema, table, timeout_ms + ) + return found + + def load_arrow( + self, + schema: str, + table: str, + reader: pa.RecordBatchReader, + columns: Sequence[tuple[str, str]], + timeout_ms: int | None = None, + ) -> int: + """Create `schema.table` from an Arrow stream, each column cast to its DuckDB type.""" + data = stream_bytes(reader.schema, reader) + found: int = self._client.call( + "load_arrow", self.handle, schema, table, data, tuple(columns), timeout_ms + ) + return found + + def cancel(self) -> None: + """Cancel from any thread: a running statement is interrupted, and the block rolls back.""" + self._client.call("cancel_transaction", self.handle) diff --git a/packages/contracts/src/orvex/contracts/store/client_sources.py b/packages/contracts/src/orvex/contracts/store/client_sources.py new file mode 100644 index 00000000..2ef47373 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/client_sources.py @@ -0,0 +1,147 @@ +"""The consumer's side of the store contract's sources besides managed lakes: an existing +DuckLake attached read-only, and a verified ClickHouse connection, each reached through a +`StoreClient` (`orvex.contracts.store.client`).""" + +from __future__ import annotations + +import datetime as dt +from collections.abc import Sequence +from typing import Any + +import pyarrow as pa +from orvex.contracts.store.client import StoreClient +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.v1 import ( + ExistingLakeAttachment, + Parameter, + ReadRequest, + Snapshot, + TablesResult, +) +from orvex.contracts.store.values import ( + Budget, + ClickHouseCatalog, + ClickHouseHandle, + ClickHouseTarget, + ExistingLakeHandle, + ExistingLakeLimits, + SourceColumn, +) + + +class ExistingLakeReads: + """What may be asked of one attached existing lake, each call in turn and under a deadline.""" + + def __init__(self, client: StoreClient, handle: ExistingLakeHandle) -> None: + self._client = client + self.handle = handle + + @property + def name(self) -> str: + return self.handle.name + + def read(self, request: ReadRequest, committed_at: dt.datetime) -> pa.RecordBatchReader: + """`request`'s statement at `request.snapshot`, which must still have been committed at + `committed_at`.""" + ticket = self._client.call("start_existing_read", self.handle, request, committed_at) + return self._client.reader(ticket) + + def table_columns(self, snapshot: Snapshot) -> TablesResult: + found: TablesResult = self._client.call("existing_table_columns", self.handle, snapshot) + return found + + def views(self, snapshot: Snapshot) -> frozenset[str]: + """`schema.view` of every view of the lake alive at `snapshot`, lower-cased.""" + return frozenset(self._client.call("existing_views", self.handle, snapshot)) + + def committed_at(self, snapshot: int) -> dt.datetime | None: + found: dt.datetime | None = self._client.call( + "existing_committed_at", self.handle, snapshot + ) + return found + + def latest(self) -> Snapshot: + found: Snapshot = self._client.call("existing_latest", self.handle) + return found + + def verify(self, snapshot: Snapshot, current: ExistingLakeAttachment) -> Snapshot: + found: Snapshot = self._client.call("existing_verify", self.handle, snapshot, current) + return found + + def close(self) -> None: + self._client.call("close_existing_lake", self.handle) + + +def open_existing_lake( + client: StoreClient, attachment: ExistingLakeAttachment, limits: ExistingLakeLimits +) -> ExistingLakeReads: + """Attach an existing DuckLake read-only, its catalog and sandbox checked first.""" + return ExistingLakeReads(client, client.call("open_existing_lake", attachment, limits)) + + +class ClickHouse: + """One verified connection to one ClickHouse database.""" + + def __init__(self, client: StoreClient, handle: ClickHouseHandle) -> None: + self._client = client + self.handle = handle + + @property + def database(self) -> str: + """The database every read names its tables in.""" + return self.handle.database + + @property + def version(self) -> str: + return self.handle.version + + def columns(self, table: str) -> tuple[SourceColumn, ...]: + found: tuple[SourceColumn, ...] = self._client.call( + "clickhouse_columns", self.handle, table + ) + return found + + def catalog(self) -> tuple[dict[str, tuple[SourceColumn, ...]], dict[str, StoreError]]: + """Every table with its columns' exact types; apart, each table holding a column with no + exact type, as the error naming that column.""" + found: ClickHouseCatalog = self._client.call("clickhouse_catalog", self.handle) + refused = { + table: StoreError( + message.code.root, + message.message, + {detail.name: detail.value for detail in message.details}, + ) + for table, message in found.refused.items() + } + return dict(found.tables), refused + + def estimate(self, sql: str, parameters: Sequence[Parameter]) -> list[dict[str, Any]]: + """`EXPLAIN ESTIMATE` of one read, checked and bound as the read is.""" + found = self._client.call("clickhouse_estimate", self.handle, sql, tuple(parameters)) + return [dict(row) for row in found] + + def now(self) -> dt.datetime: + """The server's clock: a live read's instant (ADR-0027 §5).""" + found: dt.datetime = self._client.call("clickhouse_now", self.handle) + return found + + def read( + self, + sql: str, + parameters: Sequence[Parameter], + columns: Sequence[tuple[str, str]], + timeout_ms: int | None = None, + ) -> pa.RecordBatchReader: + """Run one SELECT and return its result as Arrow in the store contract's types.""" + ticket = self._client.call( + "start_clickhouse_read", self.handle, sql, tuple(parameters), tuple(columns), timeout_ms + ) + return self._client.reader(ticket) + + def close(self) -> None: + self._client.call("close_clickhouse", self.handle) + + +def connect_clickhouse(client: StoreClient, target: ClickHouseTarget, budget: Budget) -> ClickHouse: + """Open and verify a connection: nothing is read unless every check passes.""" + return ClickHouse(client, client.call("connect_clickhouse", target, budget)) diff --git a/packages/contracts/src/orvex/contracts/store/interface.py b/packages/contracts/src/orvex/contracts/store/interface.py new file mode 100644 index 00000000..d976c6e2 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/interface.py @@ -0,0 +1,394 @@ +"""The store's interface contract (#486 §5 step 2, ADR-0033): every operation its consumers use, +by capability, with the codes each may raise (`orvex.contracts.store.errors.StoreError`). + +Each operation takes the call envelope first. Arrow crosses as IPC streams (`bytes`): a read's +schema in its handle and then one batch per `read_batch`, never a live reader. What the store +holds open (lakes, transactions, reads, connections) is named by a handle +(`orvex.contracts.store.values`); the consumer's facade is `orvex.contracts.store.client`. + +An operation changes state when it changes a lake, or what later calls see of a lake, a +transaction or a read. A ClickHouse handle only names a verified target (each call opens its +own HTTP connection and nothing in ClickHouse changes), so connecting and closing do not; +closing a handle that names nothing is harmless. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from typing import Protocol + +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.interface import InterfaceContract, Operation +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.v1 import ( + ChangesRequest, + ChangesResult, + CommitResult, + ExistingLakeAttachment, + FileSourceAttachment, + LocalLakeLocation, + MaintenanceReport, + Parameter, + PostgresLakeLocation, + PostgresSourceAttachment, + ReadRequest, + Snapshot, + SqliteLakeLocation, + SqliteSourceAttachment, + StatementRequest, + StatisticsRequest, + StatisticsResult, + TablesRequest, + TablesResult, +) +from orvex.contracts.store.values import ( + Budget, + ClickHouseCatalog, + ClickHouseHandle, + ClickHouseTarget, + ExistingLakeHandle, + ExistingLakeLimits, + Held, + JsonFrame, + LakeHandle, + Limits, + ReadHandle, + SourceColumn, + TransactionHandle, +) +from orvex.contracts.version import Version + +Location = LocalLakeLocation | SqliteLakeLocation | PostgresLakeLocation +Source = FileSourceAttachment | SqliteSourceAttachment | PostgresSourceAttachment +Columns = tuple[tuple[str, str], ...] +EstimateRow = dict[str, str | int | float | bool | None] + + +class Store(Protocol): + # --- lake ---------------------------------------------------------------------------------- + def open_lake( + self, + envelope: CallEnvelope, + location: Location, + limits: Limits, + sources: tuple[Source, ...], + ) -> LakeHandle: ... + + def close_lake(self, envelope: CallEnvelope, lake: LakeHandle) -> None: ... + + def held(self, envelope: CallEnvelope) -> Held: ... + + def current_snapshot(self, envelope: CallEnvelope, lake: LakeHandle) -> Snapshot: ... + + def snapshot( + self, envelope: CallEnvelope, lake: LakeHandle, snapshot_id: int + ) -> Snapshot | None: ... + + def tables(self, envelope: CallEnvelope, lake: LakeHandle) -> tuple[tuple[str, str], ...]: ... + + def describe( + self, + envelope: CallEnvelope, + lake: LakeHandle, + sql: str, + parameters: tuple[Parameter, ...], + ) -> Columns: ... + + def describe_source( + self, + envelope: CallEnvelope, + lake: LakeHandle, + name: str, + sql: str, + parameters: tuple[Parameter, ...], + ) -> Columns: ... + + def statistics( + self, envelope: CallEnvelope, lake: LakeHandle, request: StatisticsRequest + ) -> StatisticsResult: ... + + def changes( + self, envelope: CallEnvelope, lake: LakeHandle, request: ChangesRequest + ) -> ChangesResult: ... + + def table_columns( + self, envelope: CallEnvelope, lake: LakeHandle, request: TablesRequest + ) -> TablesResult: ... + + # --- read ---------------------------------------------------------------------------------- + def start_read( + self, envelope: CallEnvelope, lake: LakeHandle, request: ReadRequest + ) -> ReadHandle: ... + + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: ... + + def cancel_read(self, envelope: CallEnvelope, read: ReadHandle) -> None: ... + + def close_read(self, envelope: CallEnvelope, read: ReadHandle) -> None: ... + + # --- write --------------------------------------------------------------------------------- + def begin(self, envelope: CallEnvelope, lake: LakeHandle) -> TransactionHandle: ... + + def execute( + self, envelope: CallEnvelope, transaction: TransactionHandle, request: StatementRequest + ) -> None: ... + + def value( + self, envelope: CallEnvelope, transaction: TransactionHandle, request: StatementRequest + ) -> str | None: ... + + def land( + self, + envelope: CallEnvelope, + transaction: TransactionHandle, + source: str, + sql: str, + parameters: tuple[Parameter, ...], + schema: str, + table: str, + timeout_ms: int | None, + ) -> int: ... + + def load_arrow( + self, + envelope: CallEnvelope, + transaction: TransactionHandle, + schema: str, + table: str, + data: bytes, + columns: Columns, + timeout_ms: int | None, + ) -> int: ... + + def cancel_transaction( + self, envelope: CallEnvelope, transaction: TransactionHandle + ) -> None: ... + + def commit(self, envelope: CallEnvelope, transaction: TransactionHandle) -> CommitResult: ... + + def rollback(self, envelope: CallEnvelope, transaction: TransactionHandle) -> None: ... + + # --- maintenance --------------------------------------------------------------------------- + def maintain( + self, + envelope: CallEnvelope, + lake: LakeHandle, + expire_before: datetime, + orphans_before: datetime, + ) -> MaintenanceReport: ... + + # --- existing lake ------------------------------------------------------------------------- + def open_existing_lake( + self, + envelope: CallEnvelope, + attachment: ExistingLakeAttachment, + limits: ExistingLakeLimits, + ) -> ExistingLakeHandle: ... + + def close_existing_lake(self, envelope: CallEnvelope, lake: ExistingLakeHandle) -> None: ... + + def existing_latest(self, envelope: CallEnvelope, lake: ExistingLakeHandle) -> Snapshot: ... + + def existing_verify( + self, + envelope: CallEnvelope, + lake: ExistingLakeHandle, + snapshot: Snapshot, + current: ExistingLakeAttachment, + ) -> Snapshot: ... + + def existing_committed_at( + self, envelope: CallEnvelope, lake: ExistingLakeHandle, snapshot: int + ) -> datetime | None: ... + + def existing_table_columns( + self, envelope: CallEnvelope, lake: ExistingLakeHandle, snapshot: Snapshot + ) -> TablesResult: ... + + def existing_views( + self, envelope: CallEnvelope, lake: ExistingLakeHandle, snapshot: Snapshot + ) -> tuple[str, ...]: ... + + def start_existing_read( + self, + envelope: CallEnvelope, + lake: ExistingLakeHandle, + request: ReadRequest, + committed_at: datetime, + ) -> ReadHandle: ... + + # --- clickhouse ---------------------------------------------------------------------------- + def connect_clickhouse( + self, envelope: CallEnvelope, target: ClickHouseTarget, budget: Budget + ) -> ClickHouseHandle: ... + + def close_clickhouse(self, envelope: CallEnvelope, engine: ClickHouseHandle) -> None: ... + + def clickhouse_columns( + self, envelope: CallEnvelope, engine: ClickHouseHandle, table: str + ) -> tuple[SourceColumn, ...]: ... + + def clickhouse_catalog( + self, envelope: CallEnvelope, engine: ClickHouseHandle + ) -> ClickHouseCatalog: ... + + def clickhouse_estimate( + self, + envelope: CallEnvelope, + engine: ClickHouseHandle, + sql: str, + parameters: tuple[Parameter, ...], + ) -> tuple[EstimateRow, ...]: ... + + def clickhouse_now(self, envelope: CallEnvelope, engine: ClickHouseHandle) -> datetime: ... + + def start_clickhouse_read( + self, + envelope: CallEnvelope, + engine: ClickHouseHandle, + sql: str, + parameters: tuple[Parameter, ...], + columns: Columns, + timeout_ms: int | None, + ) -> ReadHandle: ... + + # --- encoding ------------------------------------------------------------------------------ + def json_frame(self, envelope: CallEnvelope, schema: bytes) -> JsonFrame: ... + + def json_rows(self, envelope: CallEnvelope, batch: bytes, first: bool) -> tuple[str, ...]: ... + + # --- separation ---------------------------------------------------------------------------- + def is_object_storage(self, envelope: CallEnvelope, location: str) -> bool: ... + + def location_components( + self, envelope: CallEnvelope, name: str, location: str + ) -> tuple[str, ...]: ... + + def locations_overlap( + self, envelope: CallEnvelope, name: str, first: str, second: str + ) -> bool: ... + + def check_apart( + self, + envelope: CallEnvelope, + name: str, + data_path: str, + catalog: str | None, + protected: tuple[str, ...], + protected_files: tuple[str, ...], + ) -> None: ... + + +# The settings a ClickHouse reader's profile must fix, each to this value, for a read to mean +# what the semantic engine wrote (the `clickhouse` capability refuses a connection without them). +REQUIRED_SETTINGS: Mapping[str, str] = { + "join_use_nulls": "1", # an outer join gives NULL, never a type's default value + "group_by_use_nulls": "1", # a subtotal's key is NULL, never a default value + "transform_null_in": "0", # NULL membership is written out by the adapter + "count_distinct_implementation": "uniqExact", # an exact distinct count + "cast_keep_nullable": "1", # CAST keeps NULL + "aggregate_functions_null_for_empty": "0", # empty-set results are written out by the adapter + "cancel_http_readonly_queries_on_client_close": "1", # closing the connection cancels +} + + +# Every code of `contracts/store/v1`: each operation reaches DuckDB, a source or a check that +# may refuse with any of them, so each declares them all. +CODES = ( + "STORE_SNAPSHOT_NOT_FOUND", + "STORE_SNAPSHOT_EXPIRED", + "STORE_STATEMENT_INVALID", + "STORE_QUERY_FAILED", + "STORE_PARAMETER_INVALID", + "STORE_LIMITS_REQUIRED", + "STORE_EXTENSION_UNAVAILABLE", + "STORE_TRANSACTION_CONFLICT", + "STORE_SECRET_MISSING", + "STORE_SECRET_INLINE", + "STORE_CATALOG_UNAVAILABLE", + "STORE_LOCATION_INVALID", + "STORE_TLS_REQUIRED", + "STORE_TYPE_UNSUPPORTED", + "STORE_TIMEOUT", + "STORE_CANCELLED", + "STORE_BUSY", + "SOURCE_NOT_FOUND", + "STORE_SOURCE_INVALID", + "STORE_SOURCE_FAILED", + "STORE_CONNECTION_FAILED", + "UNSUPPORTED_SOURCE_TYPE", + "SOURCE_NOT_READ_ONLY", + "ENGINE_VERSION_UNTESTED", + "ENGINE_SETTING_REQUIRED", +) + + +def _operation(name: str, capability: str, *, changes_state: bool = False) -> Operation: + return Operation(name, capability, CODES, changes_state=changes_state) + + +STORE = InterfaceContract( + name="store", + version=Version(1, 0), + protocol=Store, + error_type=StoreError, + capabilities=( + "lake", + "read", + "write", + "maintenance", + "existing-lake", + "clickhouse", + "encoding", + "separation", + ), + errors=CODES, + operations=( + _operation("open_lake", "lake", changes_state=True), + _operation("close_lake", "lake", changes_state=True), + _operation("held", "lake"), + _operation("current_snapshot", "lake"), + _operation("snapshot", "lake"), + _operation("tables", "lake"), + _operation("describe", "lake"), + _operation("describe_source", "lake"), + _operation("statistics", "lake"), + _operation("changes", "lake"), + _operation("table_columns", "lake"), + _operation("start_read", "read"), + _operation("read_batch", "read"), + _operation("cancel_read", "read", changes_state=True), + _operation("close_read", "read", changes_state=True), + _operation("begin", "write", changes_state=True), + _operation("execute", "write", changes_state=True), + _operation("value", "write"), + _operation("land", "write", changes_state=True), + _operation("load_arrow", "write", changes_state=True), + _operation("cancel_transaction", "write", changes_state=True), + _operation("commit", "write", changes_state=True), + _operation("rollback", "write", changes_state=True), + _operation("maintain", "maintenance", changes_state=True), + _operation("open_existing_lake", "existing-lake", changes_state=True), + _operation("close_existing_lake", "existing-lake", changes_state=True), + _operation("existing_latest", "existing-lake"), + _operation("existing_verify", "existing-lake"), + _operation("existing_committed_at", "existing-lake"), + _operation("existing_table_columns", "existing-lake"), + _operation("existing_views", "existing-lake"), + _operation("start_existing_read", "existing-lake"), + _operation("connect_clickhouse", "clickhouse"), + _operation("close_clickhouse", "clickhouse"), + _operation("clickhouse_columns", "clickhouse"), + _operation("clickhouse_catalog", "clickhouse"), + _operation("clickhouse_estimate", "clickhouse"), + _operation("clickhouse_now", "clickhouse"), + _operation("start_clickhouse_read", "clickhouse"), + _operation("json_frame", "encoding"), + _operation("json_rows", "encoding"), + _operation("is_object_storage", "separation"), + _operation("location_components", "separation"), + _operation("locations_overlap", "separation"), + _operation("check_apart", "separation"), + ), +) diff --git a/packages/contracts/src/orvex/contracts/store/kit.py b/packages/contracts/src/orvex/contracts/store/kit.py new file mode 100644 index 00000000..162dd6a7 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/kit.py @@ -0,0 +1,910 @@ +"""The store's conformance kit (#486 G1): every operation of `orvex.contracts.store.interface`, +its exactness, its fail-closed refusals and the security cases consumers rely on (the SQL guard, +the sandbox, forged handles, TLS and secrets), and one idempotency case per operation that +changes state. No skips; its case count is recorded. + +Every case runs on local folders and SQLite files only, so the kit needs no server. ClickHouse +is reached in its refusals and its handles; its answers are the store's own tests' (CI). +""" + +from __future__ import annotations + +import datetime as dt +import functools +from collections.abc import Callable +from decimal import Decimal + +import pyarrow as pa +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.kit import Idempotent, Kit, KitCase +from orvex.contracts.kit_client import KitClient +from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch, stream_bytes, stream_schema +from orvex.contracts.store.interface import STORE +from orvex.contracts.store.kit_support import ( + EXISTING_LIMITS, + LIMITS, + Scenario, + check, + commit, + drain, + existing_lake, + local_location, + open_lake, + read_request, + refused, + rows, + sqlite_source, + statement, + write, +) +from orvex.contracts.store.v1 import ( + ChangesRequest, + Parameter, + StatisticsRequest, + TablesRequest, +) +from orvex.contracts.store.values import ( + Budget, + ClickHouseHandle, + ClickHouseTarget, + ExistingLakeHandle, + LakeHandle, + Limits, + ReadHandle, + TransactionHandle, +) + +EXACT = ( + "SELECT 12345678901234567890.1234567890::DECIMAL(38,10) AS amount, " + "170141183460469231731687303715884105727::HUGEINT AS huge, " + "TIMESTAMPTZ '2026-03-29 01:30:00+00' AS at, NULL::BIGINT AS missing" +) +FORGED_LAKE = LakeHandle("lake-forged") +FORGED_READ = ReadHandle("read-forged", b"") +FORGED_TRANSACTION = TransactionHandle("transaction-forged", None) +FORGED_EXISTING = ExistingLakeHandle("existing-lake-forged", "forged") +FORGED_ENGINE = ClickHouseHandle("clickhouse-forged", "db", "") +BUDGET = Budget(timeout_ms=1_000, memory_bytes=1 << 30) +PARAMETER = "STORE_PARAMETER_INVALID" +STATEMENT = "STORE_STATEMENT_INVALID" +SOURCE = "STORE_SOURCE_INVALID" + + +def _now() -> dt.datetime: + return dt.datetime.now(dt.UTC) + + +# --- lake ---------------------------------------------------------------------------------------- + +_opening = Scenario(lambda client: local_location()) + + +def _open_call(client: KitClient, envelope: CallEnvelope) -> object: + return client.send("open_lake", envelope, _opening.of(client), LIMITS, ()) + + +def _lakes(client: KitClient) -> int: + return len(client.held().lakes) + + +def _open_refuses_missing_limits(client: KitClient) -> None: + refused( + "STORE_LIMITS_REQUIRED", lambda: client.open_lake(local_location(), Limits(None, None), ()) + ) + + +def _open_refuses_missing_source(client: KitClient) -> None: + source = sqlite_source().model_copy(update={"path": "/nonexistent/kit.sqlite"}) + refused("SOURCE_NOT_FOUND", lambda: client.open_lake(local_location(), LIMITS, (source,))) + + +_closing = Scenario(open_lake) + + +def _close_call(client: KitClient, envelope: CallEnvelope) -> object: + return client.send("close_lake", envelope, _closing.of(client)) + + +def _held_lists(client: KitClient) -> None: + lake = open_lake(client) + check(lake.id in client.held().lakes, "an open lake is held") + client.close_lake(lake) + check(lake.id not in client.held().lakes, "a closed lake is let go") + refused(PARAMETER, lambda: client.current_snapshot(lake)) + + +def _forged_lake(client: KitClient) -> None: + refused(PARAMETER, lambda: client.current_snapshot(FORGED_LAKE)) + + +def _current_after_commit(client: KitClient) -> None: + lake = open_lake(client) + committed = write(client, lake, "CREATE TABLE t AS SELECT 1 AS a") + check(client.current_snapshot(lake) == committed.snapshot, "the commit's snapshot is current") + + +def _snapshot_unknown(client: KitClient) -> None: + lake = open_lake(client) + first = commit(client, lake, "CREATE TABLE t AS SELECT 1 AS a") + check(client.snapshot(lake, first.id) == first, "a kept snapshot is found") + check(client.snapshot(lake, 999_999) is None, "an unknown snapshot is None") + check(client.snapshot(lake, -1) is None, "a negative id is None") + + +def _tables(client: KitClient) -> None: + lake = open_lake(client) + write(client, lake, 'CREATE TABLE "Readings" (a INTEGER)', "CREATE TABLE other (b INTEGER)") + found = client.tables(lake) + check(found == (("main", "other"), ("main", "readings")), f"tables, lower-cased: {found}") + + +def _describe(client: KitClient) -> None: + lake = open_lake(client) + found = client.describe(lake, "SELECT 1.50::DECIMAL(18,2) AS a, NULL::VARCHAR AS b", ()) + check(found == (("a", "DECIMAL(18,2)"), ("b", "VARCHAR")), f"described: {found}") + refused(STATEMENT, lambda: client.describe(lake, "DELETE FROM x", ())) + + +def _describe_source(client: KitClient) -> None: + lake = open_lake(client, (sqlite_source(),)) + found = client.describe_source(lake, "field", "SELECT * FROM meters", ()) + check(found == (("id", "BIGINT"), ("name", "VARCHAR")), f"source columns: {found}") + refused(SOURCE, lambda: client.describe_source(lake, "nope", "SELECT 1", ())) + + +def _statistics(client: KitClient) -> None: + lake = open_lake(client) + snapshot = commit(client, lake, "CREATE TABLE t AS SELECT * FROM range(3) AS r(a)") + request = StatisticsRequest.model_validate( + {"kind": "statistics", "tables": ["main.t"], "snapshot": snapshot.id} + ) + (table,) = client.statistics(lake, request).tables + counted = table.inlined_rows + sum(file.row_count for file in table.files) + check(counted == 3, f"three rows are counted: {table}") + unknown = StatisticsRequest.model_validate( + {"kind": "statistics", "tables": ["main.nope"], "snapshot": snapshot.id} + ) + refused(STATEMENT, lambda: client.statistics(lake, unknown)) + + +def _changes(client: KitClient) -> None: + lake = open_lake(client) + since = commit(client, lake, "CREATE TABLE t AS SELECT * FROM range(3) AS r(a)") + after = commit(client, lake, "DELETE FROM t WHERE a = 0") + request = ChangesRequest.model_validate( + {"kind": "changes", "tables": ["main.t"], "since": since.id, "snapshot": after.id} + ) + (table,) = client.changes(lake, request).tables + check(table.changed and table.removed_rows == 1, f"one row removed: {table}") + backwards = ChangesRequest.model_validate( + {"kind": "changes", "tables": ["main.t"], "since": after.id, "snapshot": since.id} + ) + refused(STATEMENT, lambda: client.changes(lake, backwards)) + + +def _table_columns(client: KitClient) -> None: + lake = open_lake(client) + first = commit(client, lake, "CREATE TABLE t (a DECIMAL(18,2) NOT NULL)") + write(client, lake, "ALTER TABLE t ADD COLUMN b VARCHAR") + (table,) = client.table_columns(lake, TablesRequest(kind="tables", snapshot=first.id)).tables + names = [(c.name, c.type) for c in table.columns] + check(names == [("a", "DECIMAL(18,2)")], f"the columns as of the snapshot: {names}") + + +# --- read ---------------------------------------------------------------------------------------- + + +def _exact(client: KitClient) -> None: + lake = open_lake(client) + write(client, lake, f"CREATE TABLE t AS {EXACT}") + (row,) = rows(client, lake, "SELECT * FROM t") + check(row["amount"] == Decimal("12345678901234567890.1234567890"), f"DECIMAL: {row}") + check(str(row["huge"]) == "170141183460469231731687303715884105727", f"HUGEINT: {row}") + check(row["at"] == dt.datetime(2026, 3, 29, 1, 30, tzinfo=dt.UTC), f"TIMESTAMPTZ: {row}") + check(row["missing"] is None, f"NULL is never zero: {row}") + + +def _read_refusals(client: KitClient) -> None: + lake = open_lake(client) + snapshot = commit(client, lake, "CREATE TABLE t AS SELECT 1 AS a").id + for sql in ("SELECT * FROM read_csv('/etc/passwd')", "DELETE FROM t", "SELECT 1; SELECT 2"): + request = read_request(sql, snapshot) + refused(STATEMENT, functools.partial(client.start_read, lake, request)) + refused( + "STORE_SNAPSHOT_NOT_FOUND", + lambda: client.start_read(lake, read_request("SELECT 1", 999_999)), + ) + + +def _read_parameters(client: KitClient) -> None: + lake = open_lake(client) + snapshot = commit(client, lake, "CREATE TABLE t AS SELECT * FROM range(5) AS r(a)").id + request = read_request("SELECT count(*) AS n FROM t WHERE a >= $low", snapshot) + bound = request.model_copy( + update={ + "parameters": [ + Parameter.model_validate({"name": "low", "type": "BIGINT", "value": "3"}) + ] + } + ) + found = drain(client, client.start_read(lake, bound)).to_pylist() + check(found == [{"n": 2}], f"the parameter is bound: {found}") + + +def _read_ends(client: KitClient) -> None: + lake = open_lake(client) + snapshot = commit(client, lake, "CREATE TABLE t AS SELECT * FROM range(3) AS r(a)").id + read = client.start_read(lake, read_request("SELECT a FROM t ORDER BY a", snapshot)) + seen = [] + while (data := client.read_batch(read)) is not None: + seen += one_batch(data).column("a").to_pylist() + check(seen == [0, 1, 2], f"every row, once: {seen}") + check(client.read_batch(read) is None, "an ended read stays ended") + client.close_read(read) + refused(PARAMETER, lambda: client.read_batch(FORGED_READ)) + + +_cancelling = Scenario( + lambda client: client.start_read( + (lake := open_lake(client)), + read_request("SELECT * FROM range(10) AS r(a)", client.current_snapshot(lake).id), + ) +) + + +def _cancelled(client: KitClient) -> int: + return len(client.held().cancelled) + + +_closing_read = Scenario( + lambda client: client.start_read( + (lake := open_lake(client)), + read_request("SELECT 1 AS a", client.current_snapshot(lake).id), + ) +) + + +def _reads(client: KitClient) -> int: + return len(client.held().reads) + + +# --- write --------------------------------------------------------------------------------------- + +_beginning = Scenario(open_lake) + + +def _transactions(client: KitClient) -> int: + return len(client.held().transactions) + + +def _in_transaction(client: KitClient) -> TransactionHandle: + lake = open_lake(client, (sqlite_source(),)) + write(client, lake, "CREATE TABLE t (a INTEGER)") + found: TransactionHandle = client.begin(lake) + return found + + +_executing = Scenario(_in_transaction) +_landing = Scenario(_in_transaction) +_loading = Scenario(_in_transaction) +_cancelling_write = Scenario(_in_transaction) + + +def _count(client: KitClient, transaction: TransactionHandle, table: str) -> str | None: + """How many rows `table` has inside the transaction, or `absent`.""" + exists = statement( + f"SELECT count(*) FROM information_schema.tables WHERE table_name = '{table}'" + ) + if client.value(transaction, exists) == "0": + return "absent" + found: str | None = client.value(transaction, statement(f"SELECT count(*) FROM {table}")) + return found + + +def _write_refusals(client: KitClient) -> None: + lake = open_lake(client) + for sql in ( + "ATTACH '/tmp/kit.db' AS other", + "COPY (SELECT 1) TO '/tmp/kit.csv'", + "SET threads = 8", + ): + transaction = client.begin(lake) + refused(STATEMENT, functools.partial(client.execute, transaction, statement(sql))) + client.rollback(transaction) + check(client.tables(lake) == (), "nothing was written") + + +def _value(client: KitClient) -> None: + lake = open_lake(client) + write(client, lake, "CREATE TABLE t AS SELECT 1.50::DECIMAL(18,2) AS a") + transaction = client.begin(lake) + check(client.value(transaction, statement("SELECT a FROM t")) == "1.50", "the first cell") + check(client.value(transaction, statement("SELECT a FROM t WHERE false")) is None, "no row") + client.rollback(transaction) + + +def _load_exact(client: KitClient) -> None: + lake = open_lake(client) + batch = pa.record_batch({"amount": pa.array([Decimal("1.25"), None], pa.decimal128(18, 2))}) + transaction = client.begin(lake) + loaded = client.load_arrow( + transaction, + "main", + "loaded", + batch_bytes(batch), + (("amount", "DECIMAL(18,2)"),), + None, + ) + client.commit(transaction) + check(loaded == 2, f"two rows loaded: {loaded}") + found = rows(client, lake, "SELECT amount FROM loaded") + check(found == [{"amount": Decimal("1.25")}, {"amount": None}], f"loaded exactly: {found}") + + +def _cancel_rolls_back(client: KitClient) -> None: + lake = open_lake(client) + transaction = client.begin(lake) + client.execute(transaction, statement("CREATE TABLE t (a INTEGER)")) + client.cancel_transaction(transaction) + refused("STORE_CANCELLED", lambda: client.commit(transaction)) + check(client.tables(lake) == (), "a cancelled transaction writes nothing") + + +def _commit_refuses_views(client: KitClient) -> None: + lake = open_lake(client) + transaction = client.begin(lake) + refused(STATEMENT, lambda: client.execute(transaction, statement("CREATE VIEW v AS SELECT 1"))) + client.rollback(transaction) + refused(PARAMETER, lambda: client.commit(FORGED_TRANSACTION)) + + +def _uncommitted(client: KitClient) -> tuple[LakeHandle, TransactionHandle]: + lake = open_lake(client) + transaction = client.begin(lake) + client.execute(transaction, statement("CREATE TABLE t (a INTEGER)")) + return lake, transaction + + +_committing = Scenario(_uncommitted) + + +def _committed(client: KitClient) -> int: + lake, _ = _committing.of(client) + current: int = client.current_snapshot(lake).id + return current + + +_rolling_back = Scenario(lambda client: client.begin(open_lake(client))) + + +def _rollback_discards(client: KitClient) -> None: + lake = open_lake(client) + transaction = client.begin(lake) + client.execute(transaction, statement("CREATE TABLE t (a INTEGER)")) + client.rollback(transaction) + check(client.tables(lake) == (), "a rolled-back transaction writes nothing") + refused(PARAMETER, lambda: client.execute(transaction, statement("SELECT 1"))) + + +# --- maintenance --------------------------------------------------------------------------------- + + +def _maintained_lake(client: KitClient) -> tuple[LakeHandle, int]: + lake = open_lake(client) + first = commit(client, lake, "CREATE TABLE t AS SELECT 1 AS a").id + write(client, lake, "INSERT INTO t VALUES (2)") + return lake, first + + +_maintaining = Scenario(_maintained_lake) + + +def _expired(client: KitClient) -> bool: + lake, first = _maintaining.of(client) + return client.snapshot(lake, first) is None + + +def _maintain_refuses_naive(client: KitClient) -> None: + lake = open_lake(client) + naive = dt.datetime(2020, 1, 1) + refused(PARAMETER, lambda: client.maintain(lake, naive, naive)) + + +# --- existing lake ------------------------------------------------------------------------------- + +_attaching = Scenario(lambda client: existing_lake(client)[1]) + + +def _existing(client: KitClient) -> int: + return len(client.held().existing_lakes) + + +_detaching = Scenario(lambda client: existing_lake(client)[0]) + + +def _existing_reads(client: KitClient) -> None: + lake, attachment = existing_lake(client) + latest = client.existing_latest(lake) + check(client.existing_verify(lake, latest, attachment) == latest, "the latest is verified") + check(client.existing_committed_at(lake, latest.id) == latest.committed_at, "its time") + check(client.existing_committed_at(lake, 999_999) is None, "an unknown snapshot has none") + (table,) = client.existing_table_columns(lake, latest).tables + check(table.table == "readings", f"its table: {table.table}") + check(client.existing_views(lake, latest) == (), "no view") + read = client.start_existing_read( + lake, read_request("SELECT amount FROM readings", latest.id), latest.committed_at + ) + found = drain(client, read).to_pylist() + check(found == [{"amount": Decimal("12.50")}], f"read exactly: {found}") + client.close_existing_lake(lake) + + +def _existing_refusals(client: KitClient) -> None: + _, attachment = existing_lake(client) + encrypted = attachment.model_copy( + update={"location": attachment.location.model_copy(update={"encrypted": True})} + ) + refused("STORE_LOCATION_INVALID", lambda: client.open_existing_lake(encrypted, EXISTING_LIMITS)) + unprotected = attachment.model_copy(update={"protected_locations": None}) + refused(SOURCE, lambda: client.open_existing_lake(unprotected, EXISTING_LIMITS)) + refused(PARAMETER, lambda: client.existing_latest(FORGED_EXISTING)) + + +def _existing_forged(operation: str) -> KitCase: + def run(client: KitClient) -> None: + lake, attachment = existing_lake(client) + latest = client.existing_latest(lake) + arguments: dict[str, tuple[object, ...]] = { + "existing_latest": (), + "existing_verify": (latest, attachment), + "existing_committed_at": (latest.id,), + "existing_table_columns": (latest,), + "existing_views": (latest,), + "start_existing_read": (read_request("SELECT 1", latest.id), latest.committed_at), + } + envelope = client.envelope(operation) + refused( + PARAMETER, + lambda: client.send(operation, envelope, FORGED_EXISTING, *arguments[operation]), + ) + + return KitCase( + f"{operation.replace('_', '-')}.forged-handle", operation, "a forged handle is refused", run + ) + + +# --- clickhouse ---------------------------------------------------------------------------------- + + +def _clickhouse_refusals(client: KitClient) -> None: + remote = ClickHouseTarget( + "clickhouse.example", 8123, "db", "reader", "KIT_UNSET_PASSWORD", tls=False + ) + refused("STORE_TLS_REQUIRED", lambda: client.connect_clickhouse(remote, BUDGET)) + local = ClickHouseTarget("localhost", 8123, "db", "reader", "KIT_UNSET_PASSWORD_486", tls=False) + refused("STORE_SECRET_MISSING", lambda: client.connect_clickhouse(local, BUDGET)) + + +def _clickhouse_close(client: KitClient) -> None: + client.close_clickhouse(FORGED_ENGINE) + client.close_clickhouse(FORGED_ENGINE) + check(client.held().clickhouse == (), "closing what is not held changes nothing") + + +def _clickhouse_forged(operation: str, *arguments: object) -> KitCase: + def run(client: KitClient) -> None: + refused( + PARAMETER, + lambda: client.send(operation, client.envelope(operation), FORGED_ENGINE, *arguments), + ) + + return KitCase( + f"{operation.replace('_', '-')}.forged-handle", operation, "a forged handle is refused", run + ) + + +# --- encoding ------------------------------------------------------------------------------------ + + +def _read_schema(client: KitClient) -> tuple[bytes, pa.RecordBatch]: + lake = open_lake(client) + write(client, lake, "CREATE TABLE t AS SELECT 1.50::DECIMAL(18,2) AS a, NULL::HUGEINT AS h") + read = client.start_read( + lake, read_request("SELECT * FROM t", client.current_snapshot(lake).id) + ) + table = drain(client, read) + return read.schema, table.to_batches()[0] + + +def _json_frame(client: KitClient) -> None: + schema, _ = _read_schema(client) + frame = client.json_frame(schema) + check(frame.tail == "]}", f"the tail closes the rows and the document: {frame.tail}") + found = "".join(frame.head) + expected = ( + '{"kind":"result_json","columns":[{"name": "a", "type": "DECIMAL(18,2)"}, ' + '{"name": "h", "type": "HUGEINT"}],"rows":[' + ) + check(found == expected, f"the head: {found}") + bare = stream_bytes(pa.schema([("x", pa.int64())])) + refused("STORE_TYPE_UNSUPPORTED", lambda: client.json_frame(bare)) + + +def _json_rows(client: KitClient) -> None: + schema, batch = _read_schema(client) + stream = batch_bytes(batch, stream_schema(schema)) + check(client.json_rows(stream, True) == ('["1.50", null]',), "exact text, NULL as null") + check(client.json_rows(stream, False) == (',["1.50", null]',), "a later row is separated") + + +# --- separation ---------------------------------------------------------------------------------- + + +def _object_storage(client: KitClient) -> None: + for location in ("s3://b/k", "gs://b/k", "az://c/k"): + check(client.is_object_storage(location), f"{location} is object storage") + for location in ("/data/lake", "http://host/x", "S3://b/k"): + check(not client.is_object_storage(location), f"{location} is not object storage") + + +def _components(client: KitClient) -> None: + found = client.location_components("c", "s3://bucket/key/") + check(found == ("object", "bucket", "key"), f"object storage: {found}") + check(client.location_components("c", "/")[0] == "local", "a local path") + for location in ("relative/path", "s3://bucket/../key", "http://host/x", "/a\x00b"): + refused(SOURCE, functools.partial(client.location_components, "c", location)) + + +def _overlap(client: KitClient) -> None: + check(client.locations_overlap("c", "s3://b/a", "s3://b/a/x"), "inside overlaps") + check(not client.locations_overlap("c", "s3://b/a", "s3://b/ab"), "a sibling does not") + + +def _apart(client: KitClient) -> None: + root = str(local_location().path) + refused(SOURCE, lambda: client.check_apart("c", root + "/data", None, (root,), ())) + client.check_apart("c", "s3://bucket/lake/", None, ("s3://bucket/other/",), ()) + + +def _case( + identifier: str, operation: str, expectation: str, run: Callable[[KitClient], None] +) -> KitCase: + return KitCase(identifier, operation, expectation, run) + + +CASES = ( + # lake + KitCase( + "open-lake.idempotent", + "open_lake", + "one key opens one lake", + Idempotent(_open_call, _lakes), + ), + _case( + "open-lake.limits-required", + "open_lake", + "a lake without every limit is refused (law 7)", + _open_refuses_missing_limits, + ), + _case( + "open-lake.source-not-found", + "open_lake", + "a missing source file is refused before anything opens", + _open_refuses_missing_source, + ), + KitCase( + "close-lake.idempotent", + "close_lake", + "one key closes once", + Idempotent(_close_call, lambda c: (_closing.of(c), _lakes(c))[1]), + ), + _case( + "held.lists-open", + "held", + "what is held is listed; a closed lake is let go and refused", + _held_lists, + ), + _case( + "current-snapshot.forged-handle", + "current_snapshot", + "a forged lake handle is refused", + _forged_lake, + ), + _case( + "current-snapshot.after-commit", + "current_snapshot", + "the snapshot a commit made is current", + _current_after_commit, + ), + _case( + "snapshot.kept-or-none", + "snapshot", + "a kept snapshot is found; an unknown or negative id is None", + _snapshot_unknown, + ), + _case("tables.lower-cased", "tables", "every table, lower-cased, in order", _tables), + _case( + "describe.binder-only", + "describe", + "a SELECT's columns from the binder; anything else refused", + _describe, + ), + _case( + "describe-source.landed-types", + "describe_source", + "a source's columns as they land; an unknown source refused", + _describe_source, + ), + _case( + "statistics.counts", + "statistics", + "every row is counted; an unknown table refused", + _statistics, + ), + _case( + "changes.removed", + "changes", + "removed rows are counted; `since` after `snapshot` refused", + _changes, + ), + _case( + "table-columns.as-of", + "table_columns", + "the columns as of the snapshot asked", + _table_columns, + ), + # read + _case( + "start-read.exact", + "start_read", + "DECIMAL, HUGEINT, TIMESTAMPTZ and NULL are read exactly", + _exact, + ), + _case( + "start-read.guarded", + "start_read", + "file readers, writes, two statements and unknown snapshots are refused", + _read_refusals, + ), + _case( + "start-read.parameters", + "start_read", + "a parameter is bound by name with its type", + _read_parameters, + ), + _case( + "read-batch.ends", + "read_batch", + "every row once, then None; a forged read is refused", + _read_ends, + ), + KitCase( + "cancel-read.idempotent", + "cancel_read", + "one key cancels once", + Idempotent(lambda c, e: c.send("cancel_read", e, _cancelling.of(c)), _cancelled), + ), + KitCase( + "close-read.idempotent", + "close_read", + "one key lets a read go once", + Idempotent( + lambda c, e: c.send("close_read", e, _closing_read.of(c)), + lambda c: (_closing_read.of(c), _reads(c))[1], + ), + ), + # write + KitCase( + "begin.idempotent", + "begin", + "one key begins one transaction", + Idempotent( + lambda c, e: c.send("begin", e, _beginning.of(c)), + lambda c: (_beginning.of(c), _transactions(c))[1], + ), + ), + KitCase( + "execute.idempotent", + "execute", + "one key inserts once", + Idempotent( + lambda c, e: c.send( + "execute", e, _executing.of(c), statement("INSERT INTO t VALUES (1)") + ), + lambda c: _count(c, _executing.of(c), "t"), + ), + ), + _case( + "execute.guarded", + "execute", + "ATTACH, COPY TO and SET are refused in a transaction", + _write_refusals, + ), + _case("value.first-cell", "value", "the first cell as text; None when there is no row", _value), + KitCase( + "land.idempotent", + "land", + "one key lands once", + Idempotent( + lambda c, e: c.send( + "land", + e, + _landing.of(c), + "field", + "SELECT * FROM meters", + (), + "main", + "landed", + None, + ), + lambda c: _count(c, _landing.of(c), "landed"), + ), + ), + KitCase( + "load-arrow.idempotent", + "load_arrow", + "one key loads once", + Idempotent( + lambda c, e: c.send( + "load_arrow", + e, + _loading.of(c), + "main", + "loaded", + batch_bytes(pa.record_batch({"a": pa.array([1, 2], pa.int64())})), + (("a", "BIGINT"),), + None, + ), + lambda c: _count(c, _loading.of(c), "loaded"), + ), + ), + _case("load-arrow.exact", "load_arrow", "a DECIMAL and a NULL are loaded exactly", _load_exact), + KitCase( + "cancel-transaction.idempotent", + "cancel_transaction", + "one key cancels once", + Idempotent( + lambda c, e: c.send("cancel_transaction", e, _cancelling_write.of(c)), + lambda c: (_cancelling_write.of(c), _cancelled(c))[1], + ), + ), + _case( + "cancel-transaction.rolls-back", + "cancel_transaction", + "a cancelled transaction never commits", + _cancel_rolls_back, + ), + KitCase( + "commit.idempotent", + "commit", + "one key commits once and answers the same", + Idempotent(lambda c, e: c.send("commit", e, _committing.of(c)[1]), _committed), + ), + _case( + "commit.guarded", + "commit", + "a view is refused; a forged transaction is refused", + _commit_refuses_views, + ), + KitCase( + "rollback.idempotent", + "rollback", + "one key rolls back once", + Idempotent( + lambda c, e: c.send("rollback", e, _rolling_back.of(c)), + lambda c: (_rolling_back.of(c), _transactions(c))[1], + ), + ), + _case( + "rollback.discards", + "rollback", + "a rolled-back transaction writes nothing and is let go", + _rollback_discards, + ), + # maintenance + KitCase( + "maintain.idempotent", + "maintain", + "one key maintains once and answers the same", + Idempotent( + lambda c, e: c.send("maintain", e, _maintaining.of(c)[0], _now(), _now()), + _expired, + ), + ), + _case( + "maintain.naive-cutoff", + "maintain", + "a cutoff without a zone is refused", + _maintain_refuses_naive, + ), + # existing lake + KitCase( + "open-existing-lake.idempotent", + "open_existing_lake", + "one key attaches once", + Idempotent( + lambda c, e: c.send("open_existing_lake", e, _attaching.of(c), EXISTING_LIMITS), + lambda c: (_attaching.of(c), _existing(c))[1], + ), + ), + _case( + "open-existing-lake.guarded", + "open_existing_lake", + "encryption and a missing protected set are refused", + _existing_refusals, + ), + KitCase( + "close-existing-lake.idempotent", + "close_existing_lake", + "one key detaches once", + Idempotent( + lambda c, e: c.send("close_existing_lake", e, _detaching.of(c)), + lambda c: (_detaching.of(c), _existing(c))[1], + ), + ), + _case( + "existing-latest.reads", + "existing_latest", + "latest, verify, committed-at, columns, views and a read, exact", + _existing_reads, + ), + _existing_forged("existing_latest"), + _existing_forged("existing_verify"), + _existing_forged("existing_committed_at"), + _existing_forged("existing_table_columns"), + _existing_forged("existing_views"), + _existing_forged("start_existing_read"), + # clickhouse + _case( + "connect-clickhouse.guarded", + "connect_clickhouse", + "plain HTTP to another machine and a missing secret are refused", + _clickhouse_refusals, + ), + _case( + "close-clickhouse.harmless", + "close_clickhouse", + "closing what is not held changes nothing", + _clickhouse_close, + ), + _clickhouse_forged("clickhouse_columns", "t"), + _clickhouse_forged("clickhouse_catalog"), + _clickhouse_forged("clickhouse_estimate", "SELECT 1", ()), + _clickhouse_forged("clickhouse_now"), + _clickhouse_forged("start_clickhouse_read", "SELECT 1", (), (("a", "INTEGER"),), None), + # encoding + _case( + "json-frame.columns", + "json_frame", + "the columns with their DuckDB types and the closing tail; no type is refused", + _json_frame, + ), + _case("json-rows.exact", "json_rows", "exact text, NULL as null, rows separated", _json_rows), + # separation + _case( + "is-object-storage.schemes", + "is_object_storage", + "s3://, gs:// and az:// only, case as written", + _object_storage, + ), + _case( + "location-components.parts", + "location_components", + "object storage and local paths by parts; anything else refused", + _components, + ), + _case( + "locations-overlap.prefix", + "locations_overlap", + "inside overlaps, a sibling with the same prefix does not", + _overlap, + ), + _case( + "check-apart.protected", + "check_apart", + "a lake inside a protected location is refused", + _apart, + ), +) + +STORE_KIT = Kit(contract=STORE, recorded=58, cases=CASES) diff --git a/packages/contracts/src/orvex/contracts/store/kit_support.py b/packages/contracts/src/orvex/contracts/store/kit_support.py new file mode 100644 index 00000000..fd2bb1e2 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/kit_support.py @@ -0,0 +1,177 @@ +"""What the store's conformance kit (`orvex.contracts.store.kit`) builds its cases from: lakes in +folders of their own, statements, reads decoded from Arrow IPC, refusals by code, and the state +an idempotency case sets up once per implementation.""" + +from __future__ import annotations + +import atexit +import shutil +import sqlite3 +import tempfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pyarrow as pa +from orvex.contracts.kit_client import KitClient +from orvex.contracts.store.arrow_ipc import one_batch, stream_schema +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.v1 import ( + CommitResult, + ExistingLakeAttachment, + LocalLakeLocation, + ReadRequest, + Snapshot, + SqliteLakeLocation, + SqliteSourceAttachment, + StatementRequest, +) +from orvex.contracts.store.values import ( + ExistingLakeHandle, + ExistingLakeLimits, + LakeHandle, + Limits, + ReadHandle, +) + +LIMITS = Limits( + memory_limit="256MB", threads=1, max_concurrent=4, queue_size=16, default_timeout_ms=60_000 +) +EXISTING_LIMITS = ExistingLakeLimits(memory_limit="256MB", threads=1, timeout_ms=60_000) +_ROOT: list[Path] = [] + + +def folder() -> Path: + """A new empty folder, removed when the process ends.""" + if not _ROOT: + _ROOT.append(Path(tempfile.mkdtemp(prefix="store-kit-"))) + atexit.register(shutil.rmtree, _ROOT[0], ignore_errors=True) + return Path(tempfile.mkdtemp(dir=_ROOT[0])) + + +def local_location() -> LocalLakeLocation: + return LocalLakeLocation(kind="lake_location", catalog="local", path=str(folder())) + + +def sqlite_source(name: str = "field") -> SqliteSourceAttachment: + """A SQLite file with one table `meters (id INTEGER, name TEXT)` of two rows.""" + path = folder() / f"{name}.sqlite" + connection = sqlite3.connect(path) + try: + connection.execute("CREATE TABLE meters (id INTEGER, name TEXT)") + connection.executemany("INSERT INTO meters VALUES (?, ?)", [(1, "north"), (2, "south")]) + connection.commit() + finally: + connection.close() + return SqliteSourceAttachment( + kind="source_attachment", name=name, type="sqlite", path=str(path) + ) + + +def statement(sql: str) -> StatementRequest: + return StatementRequest(kind="statement", sql=sql, parameters=[]) + + +def read_request(sql: str, snapshot: int) -> ReadRequest: + return ReadRequest(kind="read", sql=sql, parameters=[], snapshot=snapshot, batch_size=1024) + + +def open_lake(client: KitClient, sources: tuple[SqliteSourceAttachment, ...] = ()) -> LakeHandle: + found: LakeHandle = client.open_lake(local_location(), LIMITS, sources) + return found + + +def write(client: KitClient, lake: LakeHandle, *sqls: str) -> CommitResult: + """One transaction of `sqls`, committed.""" + transaction = client.begin(lake) + for sql in sqls: + client.execute(transaction, statement(sql)) + found: CommitResult = client.commit(transaction) + return found + + +def commit(client: KitClient, lake: LakeHandle, *sqls: str) -> Snapshot: + """The snapshot one committed transaction of `sqls` made.""" + snapshot = write(client, lake, *sqls).snapshot + check(snapshot is not None, "a commit reports the snapshot it made") + assert snapshot is not None + return snapshot + + +def drain(client: KitClient, read: ReadHandle) -> pa.Table: + """Every batch of a read, then the read let go.""" + schema = stream_schema(read.schema) + batches = [] + try: + while (data := client.read_batch(read)) is not None: + batches.append(one_batch(data)) + finally: + client.close_read(read) + return pa.Table.from_batches(batches, schema=schema) + + +def rows(client: KitClient, lake: LakeHandle, sql: str) -> list[dict[str, Any]]: + snapshot = client.current_snapshot(lake).id + table: list[dict[str, Any]] = drain( + client, client.start_read(lake, read_request(sql, snapshot)) + ).to_pylist() + return table + + +def existing_lake(client: KitClient) -> tuple[ExistingLakeHandle, ExistingLakeAttachment]: + """An existing DuckLake another system wrote (a SQLite catalog in WAL mode, one table + `main.readings` of one DECIMAL row), attached read-only.""" + root = folder() + location = SqliteLakeLocation( + kind="lake_location", + catalog="sqlite", + path=str(root / "catalog.sqlite"), + data_path=(root / "data").as_posix() + "/", + ) + owner = client.open_lake(location, LIMITS, ()) + write(client, owner, "CREATE TABLE readings AS SELECT 12.50::DECIMAL(18,2) AS amount") + client.close_lake(owner) + connection = sqlite3.connect(location.path) + try: + connection.execute("PRAGMA journal_mode=WAL") + finally: + connection.close() + attachment = ExistingLakeAttachment( + kind="existing_lake_attachment", + name="owner", + location=location, + roles_in_use=[], + protected_files=[], + credentials_in_use=[], + protected_locations=[], + ) + handle: ExistingLakeHandle = client.open_existing_lake(attachment, EXISTING_LIMITS) + return handle, attachment + + +def check(condition: bool, text: str) -> None: + if not condition: + raise AssertionError(text) + + +def refused(code: str, call: Callable[[], object]) -> StoreError: + """The store's refusal of `call` with `code`; anything else fails the case.""" + try: + call() + except StoreError as error: + check(error.code == code, f"expected {code}, got {error.code}: {error.message}") + return error + raise AssertionError(f"expected {code}, the call was accepted") + + +class Scenario: + """The state one idempotency case sets up once for the client it is run with.""" + + def __init__(self, setup: Callable[[KitClient], Any]) -> None: + self._setup = setup + self._current: tuple[KitClient, Any] | None = None + + def of(self, client: KitClient) -> Any: # noqa: ANN401 - the case's own state + if self._current is None or self._current[0] is not client: + self._current = (client, self._setup(client)) + return self._current[1] diff --git a/packages/contracts/src/orvex/contracts/store/values.py b/packages/contracts/src/orvex/contracts/store/values.py new file mode 100644 index 00000000..a5cde37f --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/values.py @@ -0,0 +1,137 @@ +"""The values the store's interface contract (`orvex.contracts.store.interface`) carries besides +the generated types of `contracts/store/v1`: limits, connection targets, columns and the +handles that name what the store holds open for a caller. + +A handle is a plain value: the store keeps the open lake, transaction, read or connection, and +the handle names it by an id the store chose (unguessable, never reused). A handle that names +nothing the store holds is refused with STORE_PARAMETER_INVALID. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from orvex.contracts.store.v1 import Snapshot, StoreError + + +@dataclass(frozen=True, slots=True) +class Limits: + """Resource limits of a lake; every field is required (law 7). + + `max_concurrent` statements (reads in progress and open transactions) run at once; up to + `queue_size` more wait in arrival order; `default_timeout_ms` applies when a request sets no + `timeout_ms`. + """ + + memory_limit: str | None + threads: int | None + max_concurrent: int | None = None + queue_size: int | None = None + default_timeout_ms: int | None = None + + +@dataclass(frozen=True, slots=True) +class ExistingLakeLimits: + """The limits of one attached existing lake: its instance's memory and threads, and the time + limit of each of its reads that sets none.""" + + memory_limit: str + threads: int + timeout_ms: int + + +@dataclass(frozen=True, slots=True) +class ClickHouseTarget: + """Where the server is and who connects; the password only as an environment variable.""" + + host: str + port: int + database: str + user: str + password_env: str + tls: bool = True + ca_file: str | None = None # absent: the operating system's authorities + + +@dataclass(frozen=True, slots=True) +class Budget: + """The most one query may use: the server's limits may be tighter, never looser.""" + + timeout_ms: int + memory_bytes: int + + +@dataclass(frozen=True, slots=True) +class SourceColumn: + name: str + engine_type: str # as ClickHouse names it + type: str # the DuckDB type it maps to + + +@dataclass(frozen=True, slots=True) +class ClickHouseCatalog: + """Every table of a ClickHouse database with its columns' exact types, in order; apart, each + table holding a column with no exact type, as the error naming that column.""" + + tables: dict[str, tuple[SourceColumn, ...]] + refused: dict[str, StoreError] + + +@dataclass(frozen=True, slots=True) +class JsonFrame: + """What a `ResultJson` document holds around its rows: the fragments before the first row + (its columns) and the one after the last.""" + + head: tuple[str, ...] + tail: str + + +@dataclass(frozen=True, slots=True) +class LakeHandle: + id: str + + +@dataclass(frozen=True, slots=True) +class TransactionHandle: + """An open write transaction and the snapshot it reads (#255).""" + + id: str + started: Snapshot | None + + +@dataclass(frozen=True, slots=True) +class ReadHandle: + """A running read: its Arrow schema as an IPC stream with no batch, then its batches one at a + time (`read_batch`).""" + + id: str + schema: bytes + + +@dataclass(frozen=True, slots=True) +class ExistingLakeHandle: + id: str + name: str + + +@dataclass(frozen=True, slots=True) +class ClickHouseHandle: + """A verified connection: the database every read names its tables in, the server's + version.""" + + id: str + database: str + version: str + + +@dataclass(frozen=True, slots=True) +class Held: + """The ids of what the store holds open now, each kind in order; the cancelled + transactions and reads among them.""" + + lakes: tuple[str, ...] + transactions: tuple[str, ...] + reads: tuple[str, ...] + existing_lakes: tuple[str, ...] + clickhouse: tuple[str, ...] + cancelled: tuple[str, ...] diff --git a/packages/server/src/orvex/server/app.py b/packages/server/src/orvex/server/app.py index 78dbaeed..c65ff984 100644 --- a/packages/server/src/orvex/server/app.py +++ b/packages/server/src/orvex/server/app.py @@ -18,13 +18,13 @@ from orvex.access.local import LocalRequest from orvex.access.server_context import Audience from orvex.contracts.http_api import v1 +from orvex.contracts.store.errors import StoreError from orvex.semantic.errors import SemanticError from orvex.server import lake_refusals, semantic_delivery, semantic_http, semantic_jobs from orvex.server.call_context import REQUEST_ID_HEADER, request_correlation_id from orvex.server.local_access import CODE_PARAMETER, LocalAccess from orvex.server.route_permissions import checked_permissions from orvex.server.versions import Versions -from orvex.store.errors import StoreError from starlette.concurrency import run_in_threadpool from starlette.exceptions import HTTPException diff --git a/packages/server/src/orvex/server/assembly.py b/packages/server/src/orvex/server/assembly.py index cd055642..fe7bd437 100644 --- a/packages/server/src/orvex/server/assembly.py +++ b/packages/server/src/orvex/server/assembly.py @@ -18,9 +18,11 @@ import importlib import inspect +import threading from collections.abc import Mapping, Sequence from importlib.resources import files +from orvex.contracts.binding import Bound from orvex.contracts.interface import InterfaceContract from orvex.contracts.manifest import Manifest, Provided, parse_manifest from orvex.contracts.registry import interface_contracts @@ -228,3 +230,37 @@ def check_installed_assembly(identity: Identity, chosen: Mapping[str, str] | Non load_implementations(identity.python_namespace, manifests), chosen, ) + + +_PROVIDERS: dict[tuple[str, str], Bound] = {} +_PROVIDERS_LOCK = threading.Lock() + + +def provider(identity: Identity, contract: str, chosen: Mapping[str, str] | None = None) -> Bound: + """The implementation of `contract` the checked assembly chose, built once per process and + reached through the in-process binding: what it holds open is shared by every consumer of + the process.""" + namespace = identity.python_namespace + with _PROVIDERS_LOCK: + found = _PROVIDERS.get((namespace, contract)) + if found is not None: + return found + manifests = installed_manifests(identity, chosen) + contracts = interface_contracts(importlib.import_module(f"{namespace}.contracts")) + implementations = load_implementations(namespace, manifests) + check_assembly(manifests, contracts, implementations, chosen) + choice = (chosen or {}).get(contract) + offered = [ + key + for key in implementations + if key[1] == contract and (choice is None or key[0] == choice) + ] + if len(offered) != 1 or contract not in contracts: + raise AssemblyRefused( + [f"ASSEMBLY_NO_IMPLEMENTATION: {contract}: {len(offered)} modules provide it"] + ) + made = implementations[offered[0]] + assert isinstance(made, type) # the assembly check refuses anything else + found = Bound(contracts[contract], made(), serialized=False) + _PROVIDERS[(namespace, contract)] = found + return found diff --git a/packages/server/src/orvex/server/cli.py b/packages/server/src/orvex/server/cli.py index 17fe107e..9d1ad760 100644 --- a/packages/server/src/orvex/server/cli.py +++ b/packages/server/src/orvex/server/cli.py @@ -19,6 +19,7 @@ import uvicorn from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.store.errors import StoreError from orvex.server import clickhouse_engines, development_switch, existing_lakes from orvex.server.admin_command import add_admin_parser, admin from orvex.server.app import LocalSemantic, create_app @@ -61,7 +62,6 @@ current_versions, installed_declarations, ) -from orvex.store.errors import StoreError LOCAL_HOST = "127.0.0.1" COMMANDS = frozenset({"validate", "run", "runs", "schedule", "maintain", "evidence", "admin"}) diff --git a/packages/server/src/orvex/server/clickhouse_adapter.py b/packages/server/src/orvex/server/clickhouse_adapter.py index ebe7a74d..59609e85 100644 --- a/packages/server/src/orvex/server/clickhouse_adapter.py +++ b/packages/server/src/orvex/server/clickhouse_adapter.py @@ -20,12 +20,12 @@ from typing import Any import pyarrow as pa +from orvex.contracts.store.client_sources import ClickHouse from orvex.contracts.store.v1 import Parameter from orvex.semantic.adapters import clickhouse as ch from orvex.semantic.adapters.duckdb import PhysicalQuery, PolicyValues from orvex.semantic.errors import PlanError from orvex.semantic.evidence import EvidenceRecord -from orvex.store.clickhouse import ClickHouse ENGINE = "clickhouse" _ZONE_CHECK = (("#different", "BIGINT"),) diff --git a/packages/server/src/orvex/server/clickhouse_engines.py b/packages/server/src/orvex/server/clickhouse_engines.py index 7381e6e9..ad34cbe2 100644 --- a/packages/server/src/orvex/server/clickhouse_engines.py +++ b/packages/server/src/orvex/server/clickhouse_engines.py @@ -2,10 +2,10 @@ ADR-0027 §4, §5, §7, §8). With the development switch on (`development_switch.clickhouse`), every `clickhouse` connection -of the served project is connected and verified when the server starts (`orvex.store.clickhouse`: -a tested version, a read-only profile, the settings that decide exact meaning, limits within -the budget below). A connection that fails verification is that error for every question on -it, never a weaker connection. With the switch off, none is, and a dataset on one is +of the served project is connected and verified when the server starts (the store contract's +`clickhouse`: a tested version, a read-only profile, the settings that decide exact meaning, +limits within the budget below). A connection that fails verification is that error for every +question on it, never a weaker connection. With the switch off, none is, and a dataset on one is `UNSUPPORTED_OPERATION` (`operation: route`). - **Questions** (`pinned`): each connection's tables as its catalog lists them now, with its @@ -26,7 +26,10 @@ from dataclasses import dataclass from orvex.contracts.project.v1 import ClickhouseConnection +from orvex.contracts.store.client_sources import ClickHouse, connect_clickhouse +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import LakeTable, TablesResult +from orvex.contracts.store.values import Budget, ClickHouseTarget from orvex.semantic.evidence import EvidenceRecord from orvex.server.clickhouse_adapter import ClickHouseAdapter from orvex.server.clickhouse_evidence import EvidenceSchedule, checked @@ -34,8 +37,7 @@ from orvex.server.project import ProjectLoadError from orvex.server.semantic_http import ProjectBusy, ProjectSource, Refused from orvex.server.semantic_model import ExternalTables -from orvex.store.clickhouse import Budget, ClickHouse, ClickHouseTarget -from orvex.store.errors import StoreError +from orvex.server.store_access import store log = logging.getLogger(__name__) POLL_SECONDS = 60.0 # how often the thread looks for a due schedule (each is at least 1h) @@ -122,6 +124,12 @@ def add(self, name: str, engine: ClickHouse | StoreError, schedule: str | None) made = EvidenceSchedule(name, schedule, lambda now: self._verified(name, now)) self._connections[name] = _Connection(engine, made) + def close(self) -> None: + """Let every verified connection go at the store.""" + for connection in self._connections.values(): + if isinstance(connection.engine, ClickHouse): + connection.engine.close() + def schedule(self, name: str) -> EvidenceSchedule: return self._connections[name].schedule @@ -208,8 +216,13 @@ def opened( budget = Budget(timeout_ms=timeout_ms, memory_bytes=memory_bytes(memory_limit)) for name, connection in sorted(clickhouse_connections(connections).items()): try: - engine: ClickHouse | StoreError = ClickHouse.connect(target(connection), budget) + engine: ClickHouse | StoreError = connect_clickhouse( + store(), target(connection), budget + ) except StoreError as error: engine = error engines.add(name, engine, connection.evidence_schedule) - yield engines + try: + yield engines + finally: + engines.close() diff --git a/packages/server/src/orvex/server/clickhouse_evidence.py b/packages/server/src/orvex/server/clickhouse_evidence.py index 6122b277..4415cbbc 100644 --- a/packages/server/src/orvex/server/clickhouse_evidence.py +++ b/packages/server/src/orvex/server/clickhouse_evidence.py @@ -19,6 +19,7 @@ from typing import Any import pyarrow as pa +from orvex.contracts.store.client_sources import ClickHouse from orvex.semantic.adapters.clickhouse import ClickHouseChecks from orvex.semantic.compiler.compiled import CompiledModel from orvex.semantic.engines import Routing @@ -26,7 +27,6 @@ from orvex.semantic.external import LIVE from orvex.semantic.ossie import sql from orvex.server.existing_lake_snapshots import period -from orvex.store.clickhouse import ClickHouse MINIMUM = dt.timedelta(hours=1) DEFAULT_SCHEDULE = "1d" diff --git a/packages/server/src/orvex/server/error_problems.py b/packages/server/src/orvex/server/error_problems.py index 2c157b10..ae462536 100644 --- a/packages/server/src/orvex/server/error_problems.py +++ b/packages/server/src/orvex/server/error_problems.py @@ -3,9 +3,9 @@ from __future__ import annotations from orvex.contracts.model_config.v1 import ModelError +from orvex.contracts.store.errors import StoreError from orvex.server.problems import Problem from orvex.server.project import ProjectLoadError -from orvex.store.errors import StoreError def _from_model_error(error: ModelError) -> Problem: diff --git a/packages/server/src/orvex/server/evidence_command.py b/packages/server/src/orvex/server/evidence_command.py index 72a831e9..aec38bd0 100644 --- a/packages/server/src/orvex/server/evidence_command.py +++ b/packages/server/src/orvex/server/evidence_command.py @@ -7,6 +7,8 @@ from pathlib import Path from typing import IO +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.semantic.ossie import sql from orvex.server import evidence_runs, lake_open, lake_place, semantic_model from orvex.server.error_problems import _load_problem, _store_problem @@ -15,8 +17,6 @@ from orvex.server.locking import ProjectLocked, project_lock from orvex.server.problems import Problem from orvex.server.project import Project, ProjectLoadError, load_project -from orvex.store.errors import StoreError -from orvex.store.lake import Lake def _evidence_after_run( diff --git a/packages/server/src/orvex/server/evidence_records.py b/packages/server/src/orvex/server/evidence_records.py index 62e097d8..ab4a491d 100644 --- a/packages/server/src/orvex/server/evidence_records.py +++ b/packages/server/src/orvex/server/evidence_records.py @@ -12,10 +12,10 @@ from collections.abc import Callable, Mapping from datetime import UTC, datetime +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ChangesRequest, ReadRequest from orvex.semantic.evidence import EvidenceRecord -from orvex.store.errors import StoreError -from orvex.store.lake import Lake RECORDS = "_meta.evidence_records" CHECKED = "_meta.evidence_checked" diff --git a/packages/server/src/orvex/server/evidence_runs.py b/packages/server/src/orvex/server/evidence_runs.py index c57686d2..ff42b92a 100644 --- a/packages/server/src/orvex/server/evidence_runs.py +++ b/packages/server/src/orvex/server/evidence_runs.py @@ -24,6 +24,8 @@ from dataclasses import dataclass from datetime import UTC, datetime +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( Parameter, ParameterType, @@ -42,8 +44,6 @@ ) from orvex.server.problems import Problem from orvex.server.semantic_model import SemanticModel -from orvex.store.errors import StoreError -from orvex.store.lake import Lake BUDGET_MS = 600_000 # per verification of a project: what is left over stays `declared` diff --git a/packages/server/src/orvex/server/existing_lake_adapter.py b/packages/server/src/orvex/server/existing_lake_adapter.py index 112d32a8..bc666fe0 100644 --- a/packages/server/src/orvex/server/existing_lake_adapter.py +++ b/packages/server/src/orvex/server/existing_lake_adapter.py @@ -21,12 +21,12 @@ from typing import Any import pyarrow as pa +from orvex.contracts.store.client_sources import ExistingLakeReads +from orvex.contracts.store.errors import SNAPSHOT_CHANGED, StoreError from orvex.contracts.store.v1 import Parameter, ReadRequest, Snapshot from orvex.semantic.adapters.duckdb import PhysicalQuery, PolicyValues, compile_plan from orvex.semantic.evidence import EvidenceRecord from orvex.server.existing_lake_snapshots import SnapshotSchedule -from orvex.store.errors import StoreError -from orvex.store.existing_lake_reads import SNAPSHOT_CHANGED, LakeReads ENGINE = "ducklake" @@ -38,7 +38,7 @@ class ExistingLakeAdapter: def __init__( self, - reads: LakeReads, + reads: ExistingLakeReads, schedule: SnapshotSchedule, engine_version: str, version: Mapping[str, object] | None = None, diff --git a/packages/server/src/orvex/server/existing_lake_snapshots.py b/packages/server/src/orvex/server/existing_lake_snapshots.py index 650d2960..9c12523e 100644 --- a/packages/server/src/orvex/server/existing_lake_snapshots.py +++ b/packages/server/src/orvex/server/existing_lake_snapshots.py @@ -31,8 +31,8 @@ from dataclasses import dataclass from typing import Literal, Protocol +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ExistingLakeAttachment, Snapshot -from orvex.store.errors import StoreError MINIMUM = dt.timedelta(minutes=5) DEFAULT_SCHEDULE = "1h" @@ -55,7 +55,7 @@ def period(text: str | None, *, minimum: dt.timedelta = dt.timedelta(0)) -> dt.t class Reads(Protocol): - """What the schedule asks of the lake (`orvex.store.existing_lake_reads.LakeReads`).""" + """What the schedule asks of the lake (`client_sources.ExistingLakeReads`).""" def latest(self) -> Snapshot: ... diff --git a/packages/server/src/orvex/server/existing_lakes.py b/packages/server/src/orvex/server/existing_lakes.py index a7fcbe76..31386b05 100644 --- a/packages/server/src/orvex/server/existing_lakes.py +++ b/packages/server/src/orvex/server/existing_lakes.py @@ -3,7 +3,7 @@ With the development switch on (`development_switch`), every `ducklake` connection of the served project is attached when the server starts: a DuckDB instance of its own per lake -(`orvex.store.existing_lake`), its guarded reads, its verified-snapshot schedule +(the store contract's existing lakes), its guarded reads, its verified-snapshot schedule (`existing_lake_snapshots`) and its adapter (`existing_lake_adapter`). With the switch off, none is, and a dataset on one is `UNSUPPORTED_OPERATION` (`operation: route`). @@ -36,7 +36,11 @@ from pathlib import Path from orvex.contracts.project.v1 import DucklakeConnection +from orvex.contracts.store.client import Lake +from orvex.contracts.store.client_sources import ExistingLakeReads, open_existing_lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ExistingLakeAttachment, Snapshot +from orvex.contracts.store.values import ExistingLakeLimits from orvex.server import existing_lake_attachments from orvex.server.existing_lake_adapter import ExistingLakeAdapter from orvex.server.existing_lake_evidence import violated @@ -44,10 +48,7 @@ from orvex.server.project import ProjectLoadError from orvex.server.semantic_http import ProjectBusy, ProjectSource, Refused from orvex.server.semantic_model import ExternalTables -from orvex.store.errors import StoreError -from orvex.store.existing_lake import ExistingLake -from orvex.store.existing_lake_reads import LakeReads -from orvex.store.lake import Lake +from orvex.server.store_access import store log = logging.getLogger(__name__) POLL_SECONDS = 30.0 # how often the thread looks for a due schedule (each is at least 5m) @@ -71,7 +72,7 @@ def ducklake_connections(connections: Mapping[str, object]) -> dict[str, Ducklak @dataclass(frozen=True, slots=True) class _Lake: - reads: LakeReads + reads: ExistingLakeReads schedule: SnapshotSchedule adapter: ExistingLakeAdapter @@ -82,7 +83,7 @@ class ExistingLakes: def __init__(self, now: Callable[[], dt.datetime] = lambda: dt.datetime.now(dt.UTC)) -> None: self._now = now self._lakes: dict[str, _Lake] = {} - self._opened: list[ExistingLake] = [] + self._opened: list[ExistingLakeReads] = [] # During a check: the served project, and its managed lake or why it could not be had. self._source: ProjectSource | None = None self._held: Lake | str | None = None @@ -93,9 +94,9 @@ def __len__(self) -> int: def add( self, name: str, - lake: ExistingLake, + lake: ExistingLakeReads, schedule: Callable[[Callable[[Snapshot], str | None]], SnapshotSchedule], - reads: LakeReads, + reads: ExistingLakeReads, engine_version: str, ) -> None: """`lake`, its reads, and its schedule made with this composition's evidence.""" @@ -234,14 +235,14 @@ def opened( try: for name, connection in sorted(ducklake_connections(connections).items()): current = existing_lake_attachments.current(directory, folder, slug, name, connection) - lake = ExistingLake(current(), memory_limit=memory_limit, threads=threads) - reads = LakeReads(lake, timeout_ms=timeout_ms) + limits = ExistingLakeLimits(memory_limit, threads, timeout_ms) + reads = open_existing_lake(store(), current(), limits) def schedule( evidence: Callable[[Snapshot], str | None], name: str = name, connection: DucklakeConnection = connection, - reads: LakeReads = reads, + reads: ExistingLakeReads = reads, current: Callable[[], ExistingLakeAttachment] = current, ) -> SnapshotSchedule: return SnapshotSchedule( @@ -254,7 +255,7 @@ def schedule( freshness=connection.freshness, ) - lakes.add(name, lake, schedule, reads, engine_version) + lakes.add(name, reads, schedule, reads, engine_version) yield lakes finally: lakes.close() diff --git a/packages/server/src/orvex/server/history.py b/packages/server/src/orvex/server/history.py index 450d017e..a47ceb71 100644 --- a/packages/server/src/orvex/server/history.py +++ b/packages/server/src/orvex/server/history.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import IO +from orvex.contracts.store.errors import StoreError from orvex.server import heartbeat, lake_open, lake_place, run_metadata from orvex.server.error_problems import _load_problem, _store_problem from orvex.server.identity import Identity @@ -13,7 +14,6 @@ from orvex.server.locking import LOCK_FILE, ProjectLocked, project_lock from orvex.server.problems import Problem from orvex.server.project import ProjectLoadError, load_project -from orvex.store.errors import StoreError from orvex.transform.run_records import history_request diff --git a/packages/server/src/orvex/server/lake_open.py b/packages/server/src/orvex/server/lake_open.py index 4150d5a1..0454d41d 100644 --- a/packages/server/src/orvex/server/lake_open.py +++ b/packages/server/src/orvex/server/lake_open.py @@ -8,11 +8,13 @@ from collections.abc import Sequence from pathlib import Path +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.values import Limits from orvex.server import lake_place from orvex.server.lake_place import Place from orvex.server.project import Project, ProjectLoadError -from orvex.store.errors import StoreError -from orvex.store.lake import Lake, Limits, open_lake +from orvex.server.store_access import store from orvex.transform.sources import Attachment DEFAULT_MEMORY = "4GB" @@ -43,7 +45,7 @@ def _open_recorded(home: Path, place: Place, attachments: Sequence[Attachment] = _refuse_missing_catalog(home) lake_place.begin(home, place) try: - lake = open_lake(place.location, _limits(), attachments) + lake = store().open_lake(place.location, _limits(), attachments) except BaseException: # Ctrl-C or any error: the record goes only when nothing was created (#181). A record # that cannot be removed stays `creating`, which binds nothing that was not created; the @@ -67,7 +69,7 @@ def open_existing(home: Path, place: Place, memory: str | None = None) -> Lake: lake_place.check_unchanged(home, place) _refuse_missing_catalog(home) lake_place.adopt(home, place) # a default lake made before #204 gets its record - lake = open_lake(place.location, _limits(memory)) + lake = store().open_lake(place.location, _limits(memory)) try: lake_place.complete(home, place) # a record written before #147 gets its endpoint except BaseException: diff --git a/packages/server/src/orvex/server/lake_refusals.py b/packages/server/src/orvex/server/lake_refusals.py index e745bcae..4582bb98 100644 --- a/packages/server/src/orvex/server/lake_refusals.py +++ b/packages/server/src/orvex/server/lake_refusals.py @@ -10,8 +10,8 @@ from __future__ import annotations +from orvex.contracts.store.errors import StoreError from orvex.semantic.errors import SemanticError -from orvex.store.errors import StoreError CODE = "STORE_SOURCE_INVALID" UNKNOWN = "checks_failed" # the reason when the store gave none diff --git a/packages/server/src/orvex/server/lake_separation.py b/packages/server/src/orvex/server/lake_separation.py index 2e3babe5..a5411fc1 100644 --- a/packages/server/src/orvex/server/lake_separation.py +++ b/packages/server/src/orvex/server/lake_separation.py @@ -12,11 +12,11 @@ The check runs both ways, across the instance, whenever a project's configuration is validated: a lake location that overlaps a protected location is refused, and so is a protected location declared later inside a lake location, in whichever order the projects were registered. The -rules themselves are the store's (`orvex.store.separation`): whole path components after -resolving, object storage by bucket and key, and no file system shared with a local data -location or a SQLite catalog. Connections with the same catalog identity (host, port, database -and schema, or one SQLite file by device and inode) and the same data location are one lake -connected twice, never an overlap. +rules themselves are the store's (its contract's `separation` capability): whole path +components after resolving, object storage by bucket and key, and no file system shared with a +local data location or a SQLite catalog. Connections with the same catalog identity (host, +port, database and schema, or one SQLite file by device and inode) and the same data location +are one lake connected twice, never an overlap. `protection` gives the store what an attachment needs: the roles in use elsewhere and the protected files and locations, built from the same set. Nothing is opened but the project files, @@ -42,6 +42,7 @@ PostgresLakeCatalog, SqliteConnection, ) +from orvex.contracts.store.errors import StoreError, separation_refused from orvex.contracts.store.v1 import ( CredentialsInUseItem, LocalLakeLocation, @@ -60,8 +61,7 @@ ProjectLoadError, load_project, ) -from orvex.store import separation -from orvex.store.errors import StoreError +from orvex.server.store_access import store from pydantic import BaseModel, ValidationError CODE = "STORE_SOURCE_INVALID" @@ -231,7 +231,7 @@ def problems(found: Instance, folder: Path) -> list[Problem]: def _own_credentials(found: Instance, lake: Lake) -> frozenset[str]: """The variables an object-storage lake's credential comes from (its catalog password aside).""" - if separation.is_object_storage(lake.data): + if store().is_object_storage(lake.data): for where, which, variables in found.secrets: if (where, which) == (lake.folder, lake.name): return variables - lake.password @@ -271,7 +271,7 @@ def protection(found: Instance, folder: Path, name: str) -> Protection: folder = folder.resolve() lake = next(item for item in found.lakes if item.folder == folder and item.name == name) if found.unreadable: - raise separation.refused(name, "another project of this instance cannot be read") + raise separation_refused(name, "another project of this instance cannot be read") places = [p for p in found.places if p.lake is None or not _one_lake(lake, p.lake)] roles = sorted({role for where, which, role in found.roles if (where, which) != (folder, name)}) variables = sorted( @@ -288,7 +288,7 @@ def protection(found: Instance, folder: Path, name: str) -> Protection: ) except ValidationError: # The root folder (a file source's glob there) or a location the contract cannot hold. - raise separation.refused( + raise separation_refused( name, "a protected location is the root folder or cannot be compared" ) from None @@ -362,8 +362,9 @@ def _one_lake(first: Lake, second: Lake) -> bool: """The same lake: same catalog identity and service, and the same data location.""" if first.identity != second.identity: return False + client = store() try: - return separation.components(first.name, first.data) == separation.components( + return client.location_components(first.name, first.data) == client.location_components( second.name, second.data ) except StoreError: @@ -375,11 +376,11 @@ def _reason(lake: Lake, place: Place, *, devices: bool) -> str | None: try: if not devices: for location in filter(None, (lake.data, lake.catalog_file)): - if separation.overlaps(lake.name, location, place.location): + if store().locations_overlap(lake.name, location, place.location): return "it overlaps a protected location" return None files = [place.location] if place.is_file else [] - separation.check_apart(lake.name, lake.data, lake.catalog_file, [place.location], files) + store().check_apart(lake.name, lake.data, lake.catalog_file, [place.location], files) except StoreError as error: return str(error.details.get("reason", error.message)) return None diff --git a/packages/server/src/orvex/server/leftovers.py b/packages/server/src/orvex/server/leftovers.py index 31a12d7b..48f3a2ae 100644 --- a/packages/server/src/orvex/server/leftovers.py +++ b/packages/server/src/orvex/server/leftovers.py @@ -5,8 +5,8 @@ from collections.abc import Sequence +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import Parameter, ParameterType, StatementRequest -from orvex.store.lake import Lake from orvex.transform.incremental import WATERMARKS from orvex.transform.models import NAME, Model diff --git a/packages/server/src/orvex/server/lineage.py b/packages/server/src/orvex/server/lineage.py index c21d95eb..415280e9 100644 --- a/packages/server/src/orvex/server/lineage.py +++ b/packages/server/src/orvex/server/lineage.py @@ -11,6 +11,8 @@ import urllib.request from pathlib import Path +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.server import run_metadata from orvex.server.identity import Identity from orvex.server.problems import Problem @@ -19,8 +21,6 @@ from orvex.server.run_metadata import _Outcome from orvex.server.run_reads import _table_columns from orvex.server.versions import module_versions -from orvex.store.errors import StoreError -from orvex.store.lake import Lake from orvex.transform import lineage from orvex.transform.incremental import Columns diff --git a/packages/server/src/orvex/server/maintenance.py b/packages/server/src/orvex/server/maintenance.py index f6db0795..2fc7a508 100644 --- a/packages/server/src/orvex/server/maintenance.py +++ b/packages/server/src/orvex/server/maintenance.py @@ -24,6 +24,8 @@ RunError, Versions, ) +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( MaintenanceReport, Parameter, @@ -36,8 +38,6 @@ from orvex.server.locking import LOCK_FILE, project_lock from orvex.server.problems import Problem from orvex.server.project import Project, ProjectLoadError, load_project -from orvex.store.errors import StoreError -from orvex.store.lake import Lake TABLE = "_meta.maintenance_runs" DEFAULT_RETENTION = "90d" diff --git a/packages/server/src/orvex/server/permission_state.py b/packages/server/src/orvex/server/permission_state.py index 2d518faf..dd0a514e 100644 --- a/packages/server/src/orvex/server/permission_state.py +++ b/packages/server/src/orvex/server/permission_state.py @@ -19,11 +19,11 @@ from collections.abc import Callable from dataclasses import dataclass +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ChangesRequest from orvex.semantic.errors import SemanticError from orvex.semantic.policies import PolicyModel -from orvex.store.errors import StoreError -from orvex.store.lake import Lake @dataclass(frozen=True, slots=True) diff --git a/packages/server/src/orvex/server/project_check.py b/packages/server/src/orvex/server/project_check.py index c4649049..6e5963d5 100644 --- a/packages/server/src/orvex/server/project_check.py +++ b/packages/server/src/orvex/server/project_check.py @@ -14,6 +14,8 @@ from pathlib import Path from orvex.contracts.model_config.v1 import Reconciliation +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( LocalLakeLocation, Parameter, @@ -32,8 +34,7 @@ from orvex.server.lake_place import Place from orvex.server.problems import Problem from orvex.server.project import Project, ProjectLoadError, load_project -from orvex.store.errors import StoreError -from orvex.store.lake import Lake, open_lake +from orvex.server.store_access import store from orvex.transform import auto_filter, incremental, model_tests, reconciliation, schedule from orvex.transform.graph import ( Graph, @@ -166,7 +167,7 @@ def _bind( try: location = LocalLakeLocation(kind="lake_location", catalog="local", path=str(scratch)) try: - lake = open_lake(location, _limits(), source_attachments(project.sources)) + lake = store().open_lake(location, _limits(), source_attachments(project.sources)) except StoreError as error: return {}, {}, {}, [_store_problem(error, ".", "")] try: diff --git a/packages/server/src/orvex/server/query_cancel.py b/packages/server/src/orvex/server/query_cancel.py index 66582d4a..027554f0 100644 --- a/packages/server/src/orvex/server/query_cancel.py +++ b/packages/server/src/orvex/server/query_cancel.py @@ -15,9 +15,8 @@ from typing import Any import pyarrow as pa -from orvex.store.errors import StoreError -from orvex.store.lake import Lake -from orvex.store.limits import ReadHandle +from orvex.contracts.store.client import Lake, ReadHandle +from orvex.contracts.store.errors import StoreError class Cancel: diff --git a/packages/server/src/orvex/server/run.py b/packages/server/src/orvex/server/run.py index d3959dff..bfa465cf 100644 --- a/packages/server/src/orvex/server/run.py +++ b/packages/server/src/orvex/server/run.py @@ -16,6 +16,8 @@ from typing import IO, Literal from orvex.contracts.run_record.v1 import RunError, RunRecord +from orvex.contracts.store.client import Transaction +from orvex.contracts.store.errors import StoreError from orvex.server import lake_place, project_check, run_metadata from orvex.server.error_problems import _load_problem, _store_message, _store_problem from orvex.server.evidence_command import _evidence_after_run @@ -30,8 +32,6 @@ from orvex.server.run_metadata import _Outcome, _project_commit, _records from orvex.server.run_reads import _committed_snapshot, _count, _since, _table_columns from orvex.server.run_steps import _check_tests, _run_step, _StepFailed -from orvex.store.errors import StoreError -from orvex.store.lake import Transaction from orvex.transform import lineage from orvex.transform.graph import Graph, plan_run from orvex.transform.incremental import Columns diff --git a/packages/server/src/orvex/server/run_reads.py b/packages/server/src/orvex/server/run_reads.py index 8bf0ce04..ec83ce5b 100644 --- a/packages/server/src/orvex/server/run_reads.py +++ b/packages/server/src/orvex/server/run_reads.py @@ -3,9 +3,9 @@ from __future__ import annotations +from orvex.contracts.store.client import Lake, Transaction +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ReadRequest, StatementRequest -from orvex.store.errors import StoreError -from orvex.store.lake import Lake, Transaction from orvex.transform import incremental from orvex.transform.incremental import Columns from orvex.transform.models import Model diff --git a/packages/server/src/orvex/server/run_steps.py b/packages/server/src/orvex/server/run_steps.py index 11efb9ec..63fd3854 100644 --- a/packages/server/src/orvex/server/run_steps.py +++ b/packages/server/src/orvex/server/run_steps.py @@ -6,10 +6,10 @@ from collections.abc import Sequence from orvex.contracts.run_record.v1 import CheckResult +from orvex.contracts.store.client import Transaction from orvex.contracts.store.v1 import StatementRequest from orvex.server.problems import Problem from orvex.server.run_metadata import _Outcome -from orvex.store.lake import Transaction from orvex.transform import incremental from orvex.transform.incremental import Count, Land, Refuse, Step, Watermark from orvex.transform.model_tests import ModelTest diff --git a/packages/server/src/orvex/server/scheduler.py b/packages/server/src/orvex/server/scheduler.py index 0319c078..8d1823cf 100644 --- a/packages/server/src/orvex/server/scheduler.py +++ b/packages/server/src/orvex/server/scheduler.py @@ -39,6 +39,7 @@ from typing import IO from zoneinfo import ZoneInfo +from orvex.contracts.store.errors import StoreError from orvex.server import ( error_problems, heartbeat, @@ -56,7 +57,6 @@ from orvex.server.project import ProjectLoadError from orvex.server.project_check import Checked from orvex.server.run import Selection -from orvex.store.errors import StoreError from orvex.transform.run_records import last_run_request, model_history_request from orvex.transform.schedule import ( LOOK_AGAIN, diff --git a/packages/server/src/orvex/server/semantic_http.py b/packages/server/src/orvex/server/semantic_http.py index 566853af..66eb068d 100644 --- a/packages/server/src/orvex/server/semantic_http.py +++ b/packages/server/src/orvex/server/semantic_http.py @@ -47,6 +47,8 @@ role_of, ) from orvex.access.storage.store import Store +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.semantic.catalog.building import catalog_for_context from orvex.semantic.errors import SemanticError from orvex.semantic.execution import Answer, Engine, answer @@ -65,9 +67,7 @@ from orvex.server.query_cancel import Cancel from orvex.server.query_slots import QuerySlots, Slot from orvex.server.semantic_model import ExternalTables -from orvex.store.errors import StoreError -from orvex.store.json_encoding import arrow_to_json -from orvex.store.lake import Lake +from orvex.server.store_access import store ARROW = "application/vnd.apache.arrow.stream" ENVELOPE_KEY = b"semantic.envelope" @@ -332,7 +332,7 @@ class Asked: context: Mapping[str, Any] = field(default_factory=dict) def json_document(self) -> dict[str, Any]: - result: dict[str, Any] = json.loads(self.found.result.wire(arrow_to_json)) + result: dict[str, Any] = json.loads(self.found.result.wire(store().arrow_to_json)) return {"result": result, "receipt": self.receipt} def arrow_table(self) -> pa.Table: @@ -402,7 +402,7 @@ def kept(data_version: str) -> Mapping[str, object] | None: engine = Engine( read=read, - encoder=arrow_to_json, + encoder=store().arrow_to_json, snapshot=pinned, engine_version=settings.engine_version, result_key=settings.result_key, diff --git a/packages/server/src/orvex/server/semantic_model.py b/packages/server/src/orvex/server/semantic_model.py index 3fae4bf3..50733cc2 100644 --- a/packages/server/src/orvex/server/semantic_model.py +++ b/packages/server/src/orvex/server/semantic_model.py @@ -19,6 +19,8 @@ from typing import Literal import yaml +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import TablesRequest, TablesResult from orvex.semantic.compiler.catalog import SourceColumn from orvex.semantic.compiler.compiled import CompiledModel, compile_model @@ -36,8 +38,6 @@ from orvex.server.engine_catalog import engine_catalog from orvex.server.problems import Problem from orvex.server.regular_file import read_regular_file -from orvex.store.errors import StoreError -from orvex.store.lake import Lake FOLDER = "semantic" POLICIES = "security.yaml" diff --git a/packages/server/src/orvex/server/store_access.py b/packages/server/src/orvex/server/store_access.py new file mode 100644 index 00000000..1f6b4084 --- /dev/null +++ b/packages/server/src/orvex/server/store_access.py @@ -0,0 +1,33 @@ +"""How the server reaches the store: a `StoreClient` on the installed implementation of the store +contract (`assembly.provider`), for the server's own caller unless a caller is given.""" + +from __future__ import annotations + +import functools +import uuid + +from orvex.contracts.store.client import Caller, StoreClient +from orvex.server.assembly import provider +from orvex.server.identity import Identity, load_identity + +STORE_CONTRACT = "store" + + +@functools.cache +def _identity() -> Identity: + return load_identity() + + +def server_caller() -> Caller: + """The server acting for itself: its own actor, the instance and its current revision.""" + return Caller( + actor=f"{_identity().slug}-server", + project_id="instance", + revision="current", + correlation_id=uuid.uuid4().hex, + ) + + +def store(caller: Caller | None = None) -> StoreClient: + """The store for `caller` (the server's own by default).""" + return StoreClient(provider(_identity(), STORE_CONTRACT), caller or server_caller()) diff --git a/packages/server/src/orvex/server/validate.py b/packages/server/src/orvex/server/validate.py index 52ed5ff2..f09a6b6b 100644 --- a/packages/server/src/orvex/server/validate.py +++ b/packages/server/src/orvex/server/validate.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import IO +from orvex.contracts.store.errors import StoreError from orvex.server import lake_open, lake_place, project_check, semantic_model from orvex.server.error_problems import _load_problem, _store_problem from orvex.server.identity import load_identity @@ -13,7 +14,6 @@ from orvex.server.problems import Problem from orvex.server.project import ProjectLoadError from orvex.server.project_check import Checked -from orvex.store.errors import StoreError from orvex.transform.graph import compiled_sql diff --git a/packages/store/pyproject.toml b/packages/store/pyproject.toml index 1d447d46..3c73bc69 100644 --- a/packages/store/pyproject.toml +++ b/packages/store/pyproject.toml @@ -17,4 +17,4 @@ where = ["src"] namespaces = true [tool.setuptools.package-data] -"orvex.store" = ["py.typed", "contracts.toml"] +"orvex.store" = ["py.typed", "contracts.toml", "module.toml"] diff --git a/packages/store/src/orvex/store/__init__.py b/packages/store/src/orvex/store/__init__.py index 25876ade..f325f973 100644 --- a/packages/store/src/orvex/store/__init__.py +++ b/packages/store/src/orvex/store/__init__.py @@ -1,7 +1,8 @@ """Store and read analytical data in DuckLake through DuckDB (contract `store` v1). -The API lives in `orvex.store.lake` (`open_lake`, `Lake`, `Limits`, `Transaction`) and -`orvex.store.errors` (`StoreError`); importing the package itself needs no dependency. +Its one public face is `orvex.store.api.Store`, the implementation of the interface contract +`orvex.contracts.store.interface.STORE` (#486); everything else is internal. Importing the +package itself needs no dependency. """ from importlib.metadata import version diff --git a/packages/store/src/orvex/store/api/__init__.py b/packages/store/src/orvex/store/api/__init__.py new file mode 100644 index 00000000..5900e01a --- /dev/null +++ b/packages/store/src/orvex/store/api/__init__.py @@ -0,0 +1,6 @@ +"""The store's one public face (#486 §2 item 1): `Store`, its implementation of the interface +contract `orvex.contracts.store.interface.STORE`; everything else in the module is internal.""" + +from orvex.store.api.service import Store + +__all__ = ["Store"] diff --git a/packages/store/src/orvex/store/api/held.py b/packages/store/src/orvex/store/api/held.py new file mode 100644 index 00000000..968d4103 --- /dev/null +++ b/packages/store/src/orvex/store/api/held.py @@ -0,0 +1,107 @@ +"""What the store's contract implementation holds open for its callers, by handle id, and the +outcome of each state-changing call by idempotency key, so a repeated key has its effect once. + +Every id is random (`secrets.token_hex`) and never reused; an id that names nothing held is +refused with STORE_PARAMETER_INVALID. The outcomes of calls on a handle are forgotten when the +handle is let go; the outcomes of calls in the store's own scope (those that open or end a +handle) are kept for the last `KEPT` such calls. What is remembered never outgrows that. +""" + +from __future__ import annotations + +import secrets +import threading +from collections import OrderedDict +from collections.abc import Callable +from dataclasses import dataclass + +from orvex.contracts.store.errors import StoreError + +STORE_SCOPE = "store" # the scope of a call that opens something: no handle exists yet +KEPT = 4096 + + +@dataclass(slots=True) +class Entry: + kind: str + value: object + owner: str | None = None # the handle this one was opened on, if any + cancelled: bool = False + + +@dataclass(frozen=True, slots=True) +class _Outcome: + value: object = None + error: BaseException | None = None + + +class Holdings: + """Thread-safe: reads, cancels and transactions arrive from any thread.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._entries: dict[str, Entry] = {} + self._outcomes: dict[str, dict[tuple[str, str], _Outcome]] = {} + self._kept: OrderedDict[tuple[str, str, str], _Outcome] = OrderedDict() + + def add(self, kind: str, value: object, owner: str | None = None) -> str: + identifier = f"{kind}-{secrets.token_hex(16)}" + with self._lock: + self._entries[identifier] = Entry(kind, value, owner) + self._outcomes[identifier] = {} + return identifier + + def entry(self, kind: str, identifier: str) -> Entry: + with self._lock: + found = self._entries.get(identifier) + if found is None or found.kind != kind: + raise StoreError( + "STORE_PARAMETER_INVALID", + f"No open {kind} has this handle.", + {"handle": kind}, + ) + return found + + def get(self, kind: str, identifier: str) -> object: + return self.entry(kind, identifier).value + + def remove(self, identifier: str) -> Entry | None: + """Let a handle go, with the outcomes of the calls made on it.""" + with self._lock: + found = self._entries.pop(identifier, None) + self._outcomes.pop(identifier, None) + return found + + def ids(self, kind: str) -> tuple[str, ...]: + with self._lock: + return tuple(sorted(i for i, e in self._entries.items() if e.kind == kind)) + + def cancelled(self) -> tuple[str, ...]: + with self._lock: + return tuple(sorted(i for i, e in self._entries.items() if e.cancelled)) + + def once[T](self, scope: str, operation: str, key: str | None, work: Callable[[], T]) -> T: + """`work`'s outcome for `key` on `scope`: run the first time, repeated after that.""" + if key is None: + return work() + name = (scope, operation, key) + with self._lock: + if scope == STORE_SCOPE: + known = self._kept.get(name) + else: + known = self._outcomes.get(scope, {}).get((operation, key)) + if known is None: + try: + known = _Outcome(value=work()) + except StoreError as error: + known = _Outcome(error=error) + with self._lock: + if scope == STORE_SCOPE: + known = self._kept.setdefault(name, known) + while len(self._kept) > KEPT: + self._kept.popitem(last=False) + elif scope in self._outcomes: + known = self._outcomes[scope].setdefault((operation, key), known) + if known.error is not None: + raise known.error + return known.value # type: ignore[return-value] diff --git a/packages/store/src/orvex/store/api/service.py b/packages/store/src/orvex/store/api/service.py new file mode 100644 index 00000000..69874945 --- /dev/null +++ b/packages/store/src/orvex/store/api/service.py @@ -0,0 +1,500 @@ +"""The store's implementation of its interface contract (`orvex.contracts.store.interface`): each +operation on what `Holdings` keeps open, every state-changing call once per idempotency key.""" + +from __future__ import annotations + +import contextlib +import datetime as dt +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +import pyarrow as pa +from orvex.contracts.envelope import CallEnvelope +from orvex.contracts.store.arrow_ipc import ( + batch_bytes, + one_batch, + stream_bytes, + stream_reader, + stream_schema, +) +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.v1 import ( + ChangesRequest, + ChangesResult, + CommitResult, + ExistingLakeAttachment, + FileSourceAttachment, + LocalLakeLocation, + MaintenanceReport, + Parameter, + PostgresLakeLocation, + PostgresSourceAttachment, + ReadRequest, + Snapshot, + SqliteLakeLocation, + SqliteSourceAttachment, + StatementRequest, + StatisticsRequest, + StatisticsResult, + TablesRequest, + TablesResult, +) +from orvex.contracts.store.values import ( + Budget, + ClickHouseCatalog, + ClickHouseHandle, + ClickHouseTarget, + ExistingLakeHandle, + ExistingLakeLimits, + Held, + JsonFrame, + LakeHandle, + Limits, + ReadHandle, + SourceColumn, + TransactionHandle, +) +from orvex.store import separation +from orvex.store.api.held import STORE_SCOPE, Holdings +from orvex.store.clickhouse import ClickHouse +from orvex.store.existing_lake import ExistingLake +from orvex.store.existing_lake_reads import LakeReads +from orvex.store.json_encoding import JSON_TAIL, json_head, json_rows +from orvex.store.lake import Lake, Transaction + +LAKE, TRANSACTION, READ, EXISTING, CLICKHOUSE = ( + "lake", + "transaction", + "read", + "existing-lake", + "clickhouse", +) + + +class _RolledBack(Exception): + """Thrown into a transaction's block to roll it back.""" + + +@dataclass(slots=True) +class _Write: + block: contextlib.AbstractContextManager[Transaction] + transaction: Transaction + + +@dataclass(slots=True) +class _Read: + reader: pa.RecordBatchReader + cancel: Callable[[], None] | None + + +@dataclass(slots=True) +class _Existing: + lake: ExistingLake + reads: LakeReads + + +class Store: + """One process's store: every lake, transaction, read and connection it holds open.""" + + def __init__(self) -> None: + self._held = Holdings() + + # --- lake ---------------------------------------------------------------------------------- + + def open_lake( + self, + envelope: CallEnvelope, + location: LocalLakeLocation | SqliteLakeLocation | PostgresLakeLocation, + limits: Limits, + sources: tuple[ + FileSourceAttachment | SqliteSourceAttachment | PostgresSourceAttachment, ... + ], + ) -> LakeHandle: + def work() -> LakeHandle: + return LakeHandle(self._held.add(LAKE, Lake(location, limits, sources))) + + return self._held.once(STORE_SCOPE, "open_lake", envelope.idempotency_key, work) + + def close_lake(self, envelope: CallEnvelope, lake: LakeHandle) -> None: + def work() -> None: + opened = self._lake(lake) + self._held.remove(lake.id) + opened.close() + + self._held.once(STORE_SCOPE, f"close_lake:{lake.id}", envelope.idempotency_key, work) + + def held(self, envelope: CallEnvelope) -> Held: + return Held( + lakes=self._held.ids(LAKE), + transactions=self._held.ids(TRANSACTION), + reads=self._held.ids(READ), + existing_lakes=self._held.ids(EXISTING), + clickhouse=self._held.ids(CLICKHOUSE), + cancelled=self._held.cancelled(), + ) + + def current_snapshot(self, envelope: CallEnvelope, lake: LakeHandle) -> Snapshot: + return self._lake(lake).current_snapshot() + + def snapshot( + self, envelope: CallEnvelope, lake: LakeHandle, snapshot_id: int + ) -> Snapshot | None: + return self._lake(lake).snapshot(snapshot_id) + + def tables(self, envelope: CallEnvelope, lake: LakeHandle) -> tuple[tuple[str, str], ...]: + return tuple(sorted(self._lake(lake).tables())) + + def describe( + self, envelope: CallEnvelope, lake: LakeHandle, sql: str, parameters: tuple[Parameter, ...] + ) -> tuple[tuple[str, str], ...]: + return tuple(self._lake(lake).describe(sql, parameters)) + + def describe_source( + self, + envelope: CallEnvelope, + lake: LakeHandle, + name: str, + sql: str, + parameters: tuple[Parameter, ...], + ) -> tuple[tuple[str, str], ...]: + return tuple(self._lake(lake).describe_source(name, sql, parameters)) + + def statistics( + self, envelope: CallEnvelope, lake: LakeHandle, request: StatisticsRequest + ) -> StatisticsResult: + return self._lake(lake).statistics(request) + + def changes( + self, envelope: CallEnvelope, lake: LakeHandle, request: ChangesRequest + ) -> ChangesResult: + return self._lake(lake).changes(request) + + def table_columns( + self, envelope: CallEnvelope, lake: LakeHandle, request: TablesRequest + ) -> TablesResult: + return self._lake(lake).table_columns(request) + + # --- read ---------------------------------------------------------------------------------- + + def start_read( + self, envelope: CallEnvelope, lake: LakeHandle, request: ReadRequest + ) -> ReadHandle: + running = self._lake(lake).start_read(request) + return self._reading(running.reader, running.cancel, lake.id) + + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: + entry = self._held.entry(READ, read.id) + held = entry.value + assert isinstance(held, _Read) + if entry.cancelled and held.cancel is None: + raise StoreError("STORE_CANCELLED", "The read was cancelled.", {}) + try: + batch = held.reader.read_next_batch() + except StopIteration: + return None + return batch_bytes(batch) + + def cancel_read(self, envelope: CallEnvelope, read: ReadHandle) -> None: + def work() -> None: + entry = self._held.entry(READ, read.id) + entry.cancelled = True + held = entry.value + assert isinstance(held, _Read) + if held.cancel is not None: + held.cancel() + + self._held.once(read.id, "cancel_read", envelope.idempotency_key, work) + + def close_read(self, envelope: CallEnvelope, read: ReadHandle) -> None: + def work() -> None: + self._held.entry(READ, read.id) + entry = self._held.remove(read.id) + if entry is not None and isinstance(entry.value, _Read): + entry.value.reader.close() + + self._held.once(STORE_SCOPE, f"close_read:{read.id}", envelope.idempotency_key, work) + + # --- write --------------------------------------------------------------------------------- + + def begin(self, envelope: CallEnvelope, lake: LakeHandle) -> TransactionHandle: + def work() -> TransactionHandle: + block = self._lake(lake).transaction() + transaction = block.__enter__() + identifier = self._held.add(TRANSACTION, _Write(block, transaction), lake.id) + return TransactionHandle(identifier, transaction.started) + + return self._held.once(lake.id, "begin", envelope.idempotency_key, work) + + def execute( + self, envelope: CallEnvelope, transaction: TransactionHandle, request: StatementRequest + ) -> None: + open_ = self._write(transaction).transaction + self._held.once( + transaction.id, "execute", envelope.idempotency_key, lambda: open_.execute(request) + ) + + def value( + self, envelope: CallEnvelope, transaction: TransactionHandle, request: StatementRequest + ) -> str | None: + return self._write(transaction).transaction.value(request) + + def land( + self, + envelope: CallEnvelope, + transaction: TransactionHandle, + source: str, + sql: str, + parameters: tuple[Parameter, ...], + schema: str, + table: str, + timeout_ms: int | None, + ) -> int: + open_ = self._write(transaction).transaction + return self._held.once( + transaction.id, + "land", + envelope.idempotency_key, + lambda: open_.land(source, sql, parameters, schema, table, timeout_ms), + ) + + def load_arrow( + self, + envelope: CallEnvelope, + transaction: TransactionHandle, + schema: str, + table: str, + data: bytes, + columns: tuple[tuple[str, str], ...], + timeout_ms: int | None, + ) -> int: + open_ = self._write(transaction).transaction + return self._held.once( + transaction.id, + "load_arrow", + envelope.idempotency_key, + lambda: open_.load_arrow( + schema, table, stream_reader(data), columns, timeout_ms=timeout_ms + ), + ) + + def cancel_transaction(self, envelope: CallEnvelope, transaction: TransactionHandle) -> None: + def work() -> None: + self._held.entry(TRANSACTION, transaction.id).cancelled = True + self._write(transaction).transaction.cancel() + + self._held.once(transaction.id, "cancel", envelope.idempotency_key, work) + + def commit(self, envelope: CallEnvelope, transaction: TransactionHandle) -> CommitResult: + def work() -> CommitResult: + held = self._write(transaction) + self._held.remove(transaction.id) + held.block.__exit__(None, None, None) + assert held.transaction.result is not None + return held.transaction.result + + return self._held.once( + STORE_SCOPE, f"commit:{transaction.id}", envelope.idempotency_key, work + ) + + def rollback(self, envelope: CallEnvelope, transaction: TransactionHandle) -> None: + def work() -> None: + held = self._write(transaction) + self._held.remove(transaction.id) + reason = _RolledBack() + held.block.__exit__(_RolledBack, reason, None) + + self._held.once(STORE_SCOPE, f"rollback:{transaction.id}", envelope.idempotency_key, work) + + # --- maintenance --------------------------------------------------------------------------- + + def maintain( + self, + envelope: CallEnvelope, + lake: LakeHandle, + expire_before: dt.datetime, + orphans_before: dt.datetime, + ) -> MaintenanceReport: + opened = self._lake(lake) + return self._held.once( + lake.id, + "maintain", + envelope.idempotency_key, + lambda: opened.maintain(expire_before, orphans_before), + ) + + # --- existing lake ------------------------------------------------------------------------- + + def open_existing_lake( + self, envelope: CallEnvelope, attachment: ExistingLakeAttachment, limits: ExistingLakeLimits + ) -> ExistingLakeHandle: + def work() -> ExistingLakeHandle: + lake = ExistingLake( + attachment, memory_limit=limits.memory_limit, threads=limits.threads + ) + try: + reads = LakeReads(lake, timeout_ms=limits.timeout_ms) + except BaseException: + lake.close() + raise + return ExistingLakeHandle(self._held.add(EXISTING, _Existing(lake, reads)), lake.name) + + return self._held.once(STORE_SCOPE, "open_existing_lake", envelope.idempotency_key, work) + + def close_existing_lake(self, envelope: CallEnvelope, lake: ExistingLakeHandle) -> None: + def work() -> None: + opened = self._existing(lake) + self._held.remove(lake.id) + opened.lake.close() + + self._held.once( + STORE_SCOPE, f"close_existing_lake:{lake.id}", envelope.idempotency_key, work + ) + + def existing_latest(self, envelope: CallEnvelope, lake: ExistingLakeHandle) -> Snapshot: + return self._existing(lake).reads.latest() + + def existing_verify( + self, + envelope: CallEnvelope, + lake: ExistingLakeHandle, + snapshot: Snapshot, + current: ExistingLakeAttachment, + ) -> Snapshot: + return self._existing(lake).reads.verify(snapshot, current) + + def existing_committed_at( + self, envelope: CallEnvelope, lake: ExistingLakeHandle, snapshot: int + ) -> dt.datetime | None: + return self._existing(lake).reads.committed_at(snapshot) + + def existing_table_columns( + self, envelope: CallEnvelope, lake: ExistingLakeHandle, snapshot: Snapshot + ) -> TablesResult: + return self._existing(lake).reads.table_columns(snapshot) + + def existing_views( + self, envelope: CallEnvelope, lake: ExistingLakeHandle, snapshot: Snapshot + ) -> tuple[str, ...]: + return tuple(sorted(self._existing(lake).reads.views(snapshot))) + + def start_existing_read( + self, + envelope: CallEnvelope, + lake: ExistingLakeHandle, + request: ReadRequest, + committed_at: dt.datetime, + ) -> ReadHandle: + reader = self._existing(lake).reads.read(request, committed_at) + return self._reading(reader, None, lake.id) + + # --- clickhouse ---------------------------------------------------------------------------- + + def connect_clickhouse( + self, envelope: CallEnvelope, target: ClickHouseTarget, budget: Budget + ) -> ClickHouseHandle: + engine = ClickHouse.connect(target, budget) + identifier = self._held.add(CLICKHOUSE, engine) + return ClickHouseHandle(identifier, engine.database, engine.version) + + def close_clickhouse(self, envelope: CallEnvelope, engine: ClickHouseHandle) -> None: + if engine.id in self._held.ids(CLICKHOUSE): + self._held.remove(engine.id) + + def clickhouse_columns( + self, envelope: CallEnvelope, engine: ClickHouseHandle, table: str + ) -> tuple[SourceColumn, ...]: + return self._clickhouse(engine).columns(table) + + def clickhouse_catalog( + self, envelope: CallEnvelope, engine: ClickHouseHandle + ) -> ClickHouseCatalog: + tables, refused = self._clickhouse(engine).catalog() + return ClickHouseCatalog( + tables=tables, refused={name: error.to_message() for name, error in refused.items()} + ) + + def clickhouse_estimate( + self, + envelope: CallEnvelope, + engine: ClickHouseHandle, + sql: str, + parameters: tuple[Parameter, ...], + ) -> tuple[dict[str, str | int | float | bool | None], ...]: + rows: list[dict[str, Any]] = self._clickhouse(engine).estimate(sql, parameters) + return tuple(rows) + + def clickhouse_now(self, envelope: CallEnvelope, engine: ClickHouseHandle) -> dt.datetime: + return self._clickhouse(engine).now() + + def start_clickhouse_read( + self, + envelope: CallEnvelope, + engine: ClickHouseHandle, + sql: str, + parameters: tuple[Parameter, ...], + columns: tuple[tuple[str, str], ...], + timeout_ms: int | None, + ) -> ReadHandle: + reader = self._clickhouse(engine).read(sql, parameters, columns, timeout_ms) + return self._reading(reader, None, engine.id) + + # --- encoding ------------------------------------------------------------------------------ + + def json_frame(self, envelope: CallEnvelope, schema: bytes) -> JsonFrame: + return JsonFrame(tuple(json_head(stream_schema(schema))), JSON_TAIL) + + def json_rows(self, envelope: CallEnvelope, batch: bytes, first: bool) -> tuple[str, ...]: + return tuple(json_rows(one_batch(batch), first)) + + # --- separation ---------------------------------------------------------------------------- + + def is_object_storage(self, envelope: CallEnvelope, location: str) -> bool: + return separation.is_object_storage(location) + + def location_components( + self, envelope: CallEnvelope, name: str, location: str + ) -> tuple[str, ...]: + return separation.components(name, location) + + def locations_overlap(self, envelope: CallEnvelope, name: str, first: str, second: str) -> bool: + return separation.overlaps(name, first, second) + + def check_apart( + self, + envelope: CallEnvelope, + name: str, + data_path: str, + catalog: str | None, + protected: tuple[str, ...], + protected_files: tuple[str, ...], + ) -> None: + separation.check_apart(name, data_path, catalog, protected, protected_files) + + # --- what is held -------------------------------------------------------------------------- + + def _lake(self, lake: LakeHandle) -> Lake: + found = self._held.get(LAKE, lake.id) + assert isinstance(found, Lake) + return found + + def _write(self, transaction: TransactionHandle) -> _Write: + found = self._held.get(TRANSACTION, transaction.id) + assert isinstance(found, _Write) + return found + + def _existing(self, lake: ExistingLakeHandle) -> _Existing: + found = self._held.get(EXISTING, lake.id) + assert isinstance(found, _Existing) + return found + + def _clickhouse(self, engine: ClickHouseHandle) -> ClickHouse: + found = self._held.get(CLICKHOUSE, engine.id) + assert isinstance(found, ClickHouse) + return found + + def _reading( + self, reader: pa.RecordBatchReader, cancel: Callable[[], None] | None, owner: str + ) -> ReadHandle: + schema = stream_bytes(reader.schema) + return ReadHandle(self._held.add(READ, _Read(reader, cancel), owner), schema) diff --git a/packages/store/src/orvex/store/catalog_reads.py b/packages/store/src/orvex/store/catalog_reads.py index 868be2a6..256520ac 100644 --- a/packages/store/src/orvex/store/catalog_reads.py +++ b/packages/store/src/orvex/store/catalog_reads.py @@ -11,8 +11,8 @@ from typing import Any import duckdb +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import TableStatistics -from orvex.store.errors import StoreError from orvex.store.lake_names import LAKE from orvex.store.sql_text import identifier as _identifier diff --git a/packages/store/src/orvex/store/clickhouse.py b/packages/store/src/orvex/store/clickhouse.py index bc0c8e02..77fbf911 100644 --- a/packages/store/src/orvex/store/clickhouse.py +++ b/packages/store/src/orvex/store/clickhouse.py @@ -27,7 +27,6 @@ import time import uuid from collections.abc import Iterator, Mapping, Sequence -from dataclasses import dataclass from datetime import UTC, datetime from decimal import Decimal from typing import Any @@ -38,8 +37,10 @@ import pyarrow.compute as pc import pyarrow.ipc from duckdb.sqltypes import DuckDBPyType +from orvex.contracts.store.errors import StoreError, register_secret +from orvex.contracts.store.interface import REQUIRED_SETTINGS from orvex.contracts.store.v1 import Parameter -from orvex.store.errors import StoreError, register_secret +from orvex.contracts.store.values import Budget, ClickHouseTarget, SourceColumn from orvex.store.nested_types import refuse_nested_uhugeint # Server versions tested (major.minor): the current LTS and the latest stable (ADR-0022). @@ -47,15 +48,6 @@ # Settings that decide what a statement means; each must have exactly this value. `readonly` is # checked first, with its own code. The adapter relies on every one of them. -REQUIRED_SETTINGS: Mapping[str, str] = { - "join_use_nulls": "1", # an outer join gives NULL, never a type's default value - "group_by_use_nulls": "1", # a subtotal's key is NULL, never a default value - "transform_null_in": "0", # NULL membership is written out by the adapter - "count_distinct_implementation": "uniqExact", # an exact distinct count - "cast_keep_nullable": "1", # CAST keeps NULL - "aggregate_functions_null_for_empty": "0", # empty-set results are written out by the adapter - "cancel_http_readonly_queries_on_client_close": "1", # closing the connection cancels -} # No result-rows or rows-read limit (Owner, 2026-11-08): every question reads all the rows it # needs, so a profile that sets one is refused as any other changed setting is. LIMIT_SETTINGS = ("max_execution_time", "max_memory_usage") @@ -121,34 +113,6 @@ } -@dataclass(frozen=True, slots=True) -class ClickHouseTarget: - """Where the server is and who connects; the password only as an environment variable.""" - - host: str - port: int - database: str - user: str - password_env: str - tls: bool = True - ca_file: str | None = None # absent: the operating system's authorities - - -@dataclass(frozen=True, slots=True) -class Budget: - """The most one query may use: the server's limits may be tighter, never looser.""" - - timeout_ms: int - memory_bytes: int - - -@dataclass(frozen=True, slots=True) -class SourceColumn: - name: str - engine_type: str # as ClickHouse names it - type: str # the DuckDB type it maps to - - def _local(host: str) -> bool: return host.strip().lower() in {"localhost", "127.0.0.1", "::1"} diff --git a/packages/store/src/orvex/store/credentials.py b/packages/store/src/orvex/store/credentials.py index 0bef01b9..fff25363 100644 --- a/packages/store/src/orvex/store/credentials.py +++ b/packages/store/src/orvex/store/credentials.py @@ -7,8 +7,8 @@ import os import re +from orvex.contracts.store.errors import StoreError, register_secret from orvex.contracts.store.v1 import PostgresSourceAttachment -from orvex.store.errors import StoreError, register_secret from orvex.store.locations import _local_connection _VARIABLE = re.compile(r"\$\{env:([A-Z_][A-Z0-9_]*)\}") diff --git a/packages/store/src/orvex/store/errors.py b/packages/store/src/orvex/store/errors.py deleted file mode 100644 index b42beade..00000000 --- a/packages/store/src/orvex/store/errors.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Errors of the store contract (`contracts/store/v1`, `ErrorCode`).""" - -from __future__ import annotations - -import json -import threading -import urllib.parse - -from orvex.contracts.store.v1 import ( - ErrorCode, - ErrorDetail, - FileSourceAttachment, - SqliteSourceAttachment, -) -from orvex.contracts.store.v1 import StoreError as StoreErrorMessage - -_SECRETS: set[str] = set() -_LOCK = threading.Lock() # registration and redaction may run in several threads - - -def register_secret(value: str) -> None: - """A value that must never appear in an error: it is replaced by `***` everywhere, in every - form it takes on its way into a message (#256): as written, escaped in an SQL string (`'` - doubled) or identifier (`"` doubled), in a libpq connection string (`\\` and `\'`), in JSON - and in a URL.""" - if value: - with _LOCK: - _SECRETS.update(_forms(value)) - - -def _forms(value: str) -> set[str]: - return { - value, - value.replace("'", "''"), - value.replace('"', '""'), - value.replace("\\", "\\\\").replace("'", "\\'"), - json.dumps(value)[1:-1], - json.dumps(value, ensure_ascii=False)[1:-1], - urllib.parse.quote(value, safe=""), - urllib.parse.quote_plus(value, safe=""), - } - - -def redact(text: str) -> str: - # The longest form first, so a form inside another never leaves part of a secret behind. - with _LOCK: - secrets = sorted(_SECRETS, key=len, reverse=True) - for secret in secrets: - text = text.replace(secret, "***") - return text - - -class StoreError(Exception): - """A store failure with a stable code from the contract. Registered secrets are redacted.""" - - def __init__(self, code: str, message: str, details: dict[str, str]) -> None: - message = redact(message) - details = {name: redact(value) for name, value in details.items()} - super().__init__(f"{code}: {message}") - # Validating through the generated type rejects a code the contract does not define. - self.code: str = ErrorCode.model_validate(code).root - self.message = message - self.details = dict(details) - - def to_message(self) -> StoreErrorMessage: - return StoreErrorMessage( - kind="store_error", - code=ErrorCode.model_validate(self.code), - message=self.message, - details=[ErrorDetail(name=name, value=value) for name, value in self.details.items()], - ) - - -def _source_not_found(source: FileSourceAttachment | SqliteSourceAttachment) -> StoreError: - return StoreError( - "SOURCE_NOT_FOUND", - f"No file matches the path of source {source.name}.", - {"source": source.name}, - ) diff --git a/packages/store/src/orvex/store/existing_lake.py b/packages/store/src/orvex/store/existing_lake.py index 6ab9c3c6..cf4c7579 100644 --- a/packages/store/src/orvex/store/existing_lake.py +++ b/packages/store/src/orvex/store/existing_lake.py @@ -58,6 +58,7 @@ from pathlib import Path import duckdb +from orvex.contracts.store.errors import StoreError, register_secret, separation_refused from orvex.contracts.store.v1 import ( AzureStorage, CredentialsInUseItem, @@ -68,7 +69,6 @@ SqliteLakeLocation, ) from orvex.store import existing_lake_guard, locations, separation -from orvex.store.errors import StoreError, register_secret from orvex.store.sql_text import literal as _quote LAKE = "lake" @@ -225,7 +225,7 @@ def __init__( "STORE_LIMITS_REQUIRED", "The lake needs a memory limit and a thread count.", {} ) if attachment.protected_locations is None: - raise separation.refused(self.name, "the instance's protected locations were not given") + raise separation_refused(self.name, "the instance's protected locations were not given") # Location checks first, and every secret registered, before anything is opened. password = "" if isinstance(location, PostgresLakeLocation): @@ -552,7 +552,7 @@ def _check_apart(self) -> None: if self._folder is None: self._folder = folder elif folder != self._folder: - raise separation.refused( + raise separation_refused( self.name, "its data folder was replaced after it was attached" ) @@ -560,9 +560,9 @@ def _take(self, current: ExistingLakeAttachment) -> None: """The protected set, roles and credentials in use as `current` gives them now; refused when it is another lake's attachment or carries no protected set.""" if current.name != self.name or current.location != self.attachment.location: - raise separation.refused(self.name, "the attachment given is not this lake's") + raise separation_refused(self.name, "the attachment given is not this lake's") if current.protected_locations is None: - raise separation.refused(self.name, "the instance's protected locations were not given") + raise separation_refused(self.name, "the instance's protected locations were not given") if isinstance(current.location, PostgresLakeLocation): self._catalog_connection(current.location, current.roles_in_use) if self._storage is not None: @@ -594,7 +594,7 @@ def verify(self, snapshot: Snapshot, current: ExistingLakeAttachment) -> Snapsho {"id": snapshot.id}, ).fetchone() if found is None or int(found[0]) != since_epoch: - raise separation.refused( + raise separation_refused( self.name, "the catalog no longer lists the snapshot with its commit time" ) tables = self._connection.execute( diff --git a/packages/store/src/orvex/store/existing_lake_guard.py b/packages/store/src/orvex/store/existing_lake_guard.py index 6d37239a..0a020571 100644 --- a/packages/store/src/orvex/store/existing_lake_guard.py +++ b/packages/store/src/orvex/store/existing_lake_guard.py @@ -25,8 +25,8 @@ from typing import Any import duckdb +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter -from orvex.store.errors import StoreError from orvex.store.lake_names import LAKE from orvex.store.parameters import _bound, _typed from orvex.store.read_tree import _is_lake_table, _read_tree, _scoped diff --git a/packages/store/src/orvex/store/existing_lake_reads.py b/packages/store/src/orvex/store/existing_lake_reads.py index af259872..ac766724 100644 --- a/packages/store/src/orvex/store/existing_lake_reads.py +++ b/packages/store/src/orvex/store/existing_lake_reads.py @@ -32,19 +32,16 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.errors import FILE_REFUSED, SNAPSHOT_CHANGED, StoreError from orvex.contracts.store.v1 import ExistingLakeAttachment, ReadRequest, Snapshot, TablesResult from orvex.store.arrow_exact import _utc_schema, _with_duckdb_types, exact_select from orvex.store.catalog_tables import one_row, tables_at, tables_with_columns -from orvex.store.errors import StoreError from orvex.store.existing_lake import _EPOCH, LAKE, ExistingLake from orvex.store.existing_lake_guard import guarded from orvex.store.limits import Guard from orvex.store.nested_types import refuse_nested_uhugeint from orvex.store.sql_text import literal as _quote -SNAPSHOT_CHANGED = "snapshot_changed" -FILE_REFUSED = "file_refused" # the sandbox refused a file the lake named - class LakeReads: """What may be asked of one attached existing lake, each call in turn and under a deadline diff --git a/packages/store/src/orvex/store/file_names.py b/packages/store/src/orvex/store/file_names.py index 04fbb59d..bc9e7701 100644 --- a/packages/store/src/orvex/store/file_names.py +++ b/packages/store/src/orvex/store/file_names.py @@ -12,7 +12,7 @@ from collections.abc import Iterator import duckdb -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError from orvex.store.sql_parser import _parser _GLOB = re.compile(r"[*?\[]") diff --git a/packages/store/src/orvex/store/file_sources.py b/packages/store/src/orvex/store/file_sources.py index fd436c98..5c494e07 100644 --- a/packages/store/src/orvex/store/file_sources.py +++ b/packages/store/src/orvex/store/file_sources.py @@ -29,7 +29,7 @@ from pathlib import Path import duckdb -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError from orvex.store.sql_text import identifier, literal # Logical Parquet annotations DuckDB v2.0.0 reads correctly without a converted twin (measured, diff --git a/packages/store/src/orvex/store/isolation.py b/packages/store/src/orvex/store/isolation.py index 80a55c02..56395c1d 100644 --- a/packages/store/src/orvex/store/isolation.py +++ b/packages/store/src/orvex/store/isolation.py @@ -9,6 +9,7 @@ from pathlib import Path import duckdb +from orvex.contracts.store.errors import StoreError, source_not_found from orvex.contracts.store.v1 import ( AzureStorage, FileSourceAttachment, @@ -18,7 +19,6 @@ S3Storage, SqliteLakeLocation, ) -from orvex.store.errors import StoreError, _source_not_found from orvex.store.file_sources import view_query from orvex.store.lake_names import LAKE from orvex.store.locations import AnyLakeLocation, _sandbox_folder, _Storage, _with_root_cert @@ -75,7 +75,7 @@ def attach_file_source(self, source: FileSourceAttachment) -> list[str]: matched = self._one("SELECT list(file ORDER BY file) FROM glob($path)", {"path": path}) matched_files = matched[0] if matched else None if not isinstance(matched_files, list) or not matched_files: - raise _source_not_found(source) + raise source_not_found(source) # DuckDB's readers expand `*`, `?` and `[` in each listed name again: `r[1].parquet` # would be read as `r1.parquet` (measured on 2.0.0). Such a name is refused, as #82 # specifies; a bracket class (`r[[]1].parquet`) would read it literally (#232). diff --git a/packages/store/src/orvex/store/json_encoding.py b/packages/store/src/orvex/store/json_encoding.py index 97d10ee5..44fb41f6 100644 --- a/packages/store/src/orvex/store/json_encoding.py +++ b/packages/store/src/orvex/store/json_encoding.py @@ -11,11 +11,11 @@ import decimal import json import math -from collections.abc import Callable, Iterator +from collections.abc import Callable import pyarrow as pa import pyarrow.compute as pc -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError type JsonValue = str | int | float | bool | None @@ -206,24 +206,36 @@ def _converter(kind: str, column: str) -> Callable[[pa.Array], list[JsonValue]]: ) -def arrow_to_json(reader: pa.RecordBatchReader) -> Iterator[str]: - """The `ResultJson` document as text fragments, one batch of rows at a time. +def json_head(schema: pa.Schema) -> list[str]: + """The `ResultJson` document's fragments before its first row: its columns. - Unsupported types are reported before any row is produced. + Unsupported types are reported here, before any row is produced. """ - schema = reader.schema kinds = duckdb_types(schema) - converters = [_converter(kind, field.name) for kind, field in zip(kinds, schema, strict=True)] + for kind, field in zip(kinds, schema, strict=True): + _converter(kind, field.name) columns = [ {"name": field.name, "type": kind} for field, kind in zip(schema, kinds, strict=True) ] - yield '{"kind":"result_json","columns":' + json.dumps(columns, ensure_ascii=False) - yield ',"rows":[' - first = True - for batch in reader: - encoded = [convert(batch.column(index)) for index, convert in enumerate(converters)] - for row in zip(*encoded, strict=True): - prefix = "" if first else "," - first = False - yield prefix + json.dumps(list(row), ensure_ascii=False, allow_nan=False) - yield "]}" + return [ + '{"kind":"result_json","columns":' + json.dumps(columns, ensure_ascii=False), + ',"rows":[', + ] + + +def json_rows(batch: pa.RecordBatch, first: bool) -> list[str]: + """The fragments of one batch's rows; `first` when no row of the document came before.""" + kinds = duckdb_types(batch.schema) + converters = [ + _converter(kind, field.name) for kind, field in zip(kinds, batch.schema, strict=True) + ] + encoded = [convert(batch.column(index)) for index, convert in enumerate(converters)] + fragments = [] + for row in zip(*encoded, strict=True): + prefix = "" if first else "," + first = False + fragments.append(prefix + json.dumps(list(row), ensure_ascii=False, allow_nan=False)) + return fragments + + +JSON_TAIL = "]}" # closes the rows and the document `json_head` opened diff --git a/packages/store/src/orvex/store/lake.py b/packages/store/src/orvex/store/lake.py index dfab142b..caa08302 100644 --- a/packages/store/src/orvex/store/lake.py +++ b/packages/store/src/orvex/store/lake.py @@ -24,6 +24,7 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.errors import StoreError, source_not_found from orvex.contracts.store.v1 import ( ChangesRequest, ChangesResult, @@ -45,13 +46,13 @@ TablesRequest, TablesResult, ) +from orvex.contracts.store.values import Limits as Limits from orvex.store import source_worker, transaction from orvex.store.arrow_exact import exact_select as exact_select from orvex.store.catalog_reads import table_changes, table_statistics from orvex.store.catalog_tables import tables_at, tables_with_columns from orvex.store.catalog_values import _snapshot_of from orvex.store.credentials import _source_password -from orvex.store.errors import StoreError, _source_not_found from orvex.store.file_names import FOLDER_COST as FOLDER_COST from orvex.store.file_names import PATTERN_BUDGET as PATTERN_BUDGET from orvex.store.file_names import STEP_BUDGET as STEP_BUDGET @@ -66,7 +67,6 @@ from orvex.store.limits import MAX_TIMEOUT_MS as MAX_TIMEOUT_MS from orvex.store.limits import Admission as Admission from orvex.store.limits import Guard as Guard -from orvex.store.limits import Limits as Limits from orvex.store.limits import ReadHandle as ReadHandle from orvex.store.limits import _bounded, _require_limits from orvex.store.locations import ( @@ -208,7 +208,7 @@ def __init__( if not isinstance(source, PostgresSourceAttachment) and not ( Path(source.path).is_file() ): - raise _source_not_found(source) + raise source_not_found(source) if isinstance(source, SqliteSourceAttachment): # One consistent copy per open lake, WAL included (#126): every read of # this lake sees the file as it was now. Pages only: no SQL is parsed. diff --git a/packages/store/src/orvex/store/lake_names.py b/packages/store/src/orvex/store/lake_names.py index b12f2072..c323ab76 100644 --- a/packages/store/src/orvex/store/lake_names.py +++ b/packages/store/src/orvex/store/lake_names.py @@ -4,7 +4,7 @@ from __future__ import annotations -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError LAKE = "lake" METADATA_PREFIX = "__ducklake_metadata" diff --git a/packages/store/src/orvex/store/limits.py b/packages/store/src/orvex/store/limits.py index bee530e1..b0be400c 100644 --- a/packages/store/src/orvex/store/limits.py +++ b/packages/store/src/orvex/store/limits.py @@ -13,27 +13,12 @@ import duckdb import pyarrow as pa -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.values import Limits MAX_TIMEOUT_MS = 3_600_000 -@dataclass(frozen=True, slots=True) -class Limits: - """Resource limits of a lake; every field is required (law 7). - - `max_concurrent` statements (reads in progress and open transactions) run at once; up to - `queue_size` more wait in arrival order; `default_timeout_ms` applies when a request sets no - `timeout_ms`. - """ - - memory_limit: str | None - threads: int | None - max_concurrent: int | None = None - queue_size: int | None = None - default_timeout_ms: int | None = None - - def _require_limits(limits: Limits) -> tuple[str, int, int, int]: """The memory limit, thread count, concurrency limit and queue size, once every limit is set and within its range (law 7); `STORE_LIMITS_REQUIRED` otherwise.""" diff --git a/packages/store/src/orvex/store/locations.py b/packages/store/src/orvex/store/locations.py index bd48ca9e..4e382801 100644 --- a/packages/store/src/orvex/store/locations.py +++ b/packages/store/src/orvex/store/locations.py @@ -10,6 +10,7 @@ from dataclasses import dataclass from pathlib import Path +from orvex.contracts.store.errors import StoreError, register_secret from orvex.contracts.store.v1 import ( AzureStorage, GcsStorage, @@ -19,7 +20,6 @@ S3Storage, SqliteLakeLocation, ) -from orvex.store.errors import StoreError, register_secret # Any password key (password=, sslpassword=, ?password=) and any URI form (user:pw@host) is # refused: the password may only come from `password_env`. diff --git a/packages/store/src/orvex/store/maintenance.py b/packages/store/src/orvex/store/maintenance.py index 7d485c19..9596bcfd 100644 --- a/packages/store/src/orvex/store/maintenance.py +++ b/packages/store/src/orvex/store/maintenance.py @@ -5,9 +5,9 @@ from __future__ import annotations import duckdb +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import MaintenanceReport from orvex.store.catalog_values import _single_value, _snapshot_of -from orvex.store.errors import StoreError from orvex.store.lake_names import LAKE diff --git a/packages/store/src/orvex/store/module.toml b/packages/store/src/orvex/store/module.toml new file mode 100644 index 00000000..8c296210 --- /dev/null +++ b/packages/store/src/orvex/store/module.toml @@ -0,0 +1,13 @@ +# The manifest of this module (#486 §2.5, orvex.contracts.manifest): it provides the store +# interface contract in full and requires none. +module = "store" + +[dependencies] +runtime = ["duckdb", "orvex-contracts", "pyarrow"] +tests = ["hypothesis", "moto", "pytest"] + +[[provides]] +contract = "store" +version = "1.0" +capabilities = ["lake", "read", "write", "maintenance", "existing-lake", "clickhouse", "encoding", "separation"] +implementation = "Store" diff --git a/packages/store/src/orvex/store/nested_types.py b/packages/store/src/orvex/store/nested_types.py index 3e78cb8d..8f6814d1 100644 --- a/packages/store/src/orvex/store/nested_types.py +++ b/packages/store/src/orvex/store/nested_types.py @@ -10,7 +10,7 @@ from collections.abc import Sequence from duckdb.sqltypes import DuckDBPyType -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError _NESTED = frozenset({"list", "array", "struct", "map", "union"}) diff --git a/packages/store/src/orvex/store/parameters.py b/packages/store/src/orvex/store/parameters.py index d28eb648..b61e1fb8 100644 --- a/packages/store/src/orvex/store/parameters.py +++ b/packages/store/src/orvex/store/parameters.py @@ -9,8 +9,8 @@ import re import duckdb +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import Parameter -from orvex.store.errors import StoreError from orvex.store.sql_parser import _tokens INTEGER_RANGES = { diff --git a/packages/store/src/orvex/store/read_stream.py b/packages/store/src/orvex/store/read_stream.py index 31d192ae..7c4c98d2 100644 --- a/packages/store/src/orvex/store/read_stream.py +++ b/packages/store/src/orvex/store/read_stream.py @@ -10,9 +10,9 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.errors import StoreError from orvex.store import nested_types from orvex.store.arrow_exact import _utc_schema, _with_duckdb_types, exact_select -from orvex.store.errors import StoreError from orvex.store.limits import Guard, ReadHandle diff --git a/packages/store/src/orvex/store/read_tree.py b/packages/store/src/orvex/store/read_tree.py index 2fb73e71..2c36818e 100644 --- a/packages/store/src/orvex/store/read_tree.py +++ b/packages/store/src/orvex/store/read_tree.py @@ -9,7 +9,7 @@ from collections.abc import Iterator from typing import Any -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError from orvex.store.lake_names import LAKE from orvex.store.sql_guard import _EXTERNAL_SQL_FUNCTIONS, READ_TABLE_FUNCTIONS from orvex.store.sql_parser import _parser diff --git a/packages/store/src/orvex/store/separation.py b/packages/store/src/orvex/store/separation.py index c1bb7473..f14e9a18 100644 --- a/packages/store/src/orvex/store/separation.py +++ b/packages/store/src/orvex/store/separation.py @@ -47,7 +47,7 @@ from collections.abc import Iterable, Sequence from pathlib import Path -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import separation_refused from orvex.store.locations import azure_pairs OBJECT_STORAGE = ("s3://", "gs://", "az://") @@ -63,15 +63,6 @@ type Mount = tuple[int, Path, str] # device, mount point, file-system type -def refused(name: str, reason: str) -> StoreError: - return StoreError( - "STORE_SOURCE_INVALID", - f"Connection {name}: the lake cannot be shown apart from the instance's other files: " - f"{reason}.", - {"connection": name, "reason": reason}, - ) - - def is_object_storage(location: str) -> bool: return location.startswith(OBJECT_STORAGE) @@ -86,20 +77,24 @@ def components(name: str, location: str) -> tuple[str, ...]: if parts[-1] == "": parts.pop() if not parts or any(part in {"", ".", ".."} for part in parts): - raise refused(name, "an object-storage location has an empty, . or .. segment") + raise separation_refused( + name, "an object-storage location has an empty, . or .. segment" + ) if scheme == "az://" and "." in parts[0]: # A container name has no dot: this one names an account's endpoint. - raise refused(name, "an Azure location names an account instead of a container") + raise separation_refused( + name, "an Azure location names an account instead of a container" + ) return ("object", *parts) if _URL.match(location) or not os.path.isabs(location): - raise refused(name, "a location is neither an absolute path nor object storage") + raise separation_refused(name, "a location is neither an absolute path nor object storage") return ("local", *Path(os.path.realpath(location)).parts) def plain(name: str, path: str) -> str: """`path`, refused if it holds a NUL character (no file system path can).""" if "\x00" in path: - raise refused(name, "a location holds a NUL character") + raise separation_refused(name, "a location holds a NUL character") return path @@ -126,9 +121,9 @@ def check_apart( plain(name, path) for location in protected: if overlaps(name, data_path, location): - raise refused(name, "its data location overlaps a protected location") + raise separation_refused(name, "its data location overlaps a protected location") if catalog is not None and overlaps(name, catalog, location): - raise refused(name, "its SQLite catalog lies in a protected location") + raise separation_refused(name, "its SQLite catalog lies in a protected location") local_data = not is_object_storage(data_path) if not local_data and catalog is None: return @@ -141,7 +136,7 @@ def check_apart( for suffix in ("-wal", "-shm", "-journal", ".wal"): devices |= _device_if_present(name, path + suffix) if local_data and own_device(name, data_path, table) in devices: - raise refused( + raise separation_refused( name, "its data location is on a file system that holds a protected location, so a hard " "link there could reach a protected file", @@ -149,15 +144,19 @@ def check_apart( if catalog is not None: holding = _holding(Path(os.path.realpath(catalog)), table) if holding is None or holding[2] not in LOCAL_FILE_SYSTEMS: - raise refused(name, f"its SQLite catalog is on {_kind(holding)}{_LOCAL_ONLY}") + raise separation_refused( + name, f"its SQLite catalog is on {_kind(holding)}{_LOCAL_ONLY}" + ) for path in (catalog, *(catalog + suffix for suffix in ("-wal", "-shm", "-journal"))): found = _device_if_present(name, path) if path == catalog and not found: - raise refused(name, "its SQLite catalog does not exist") + raise separation_refused(name, "its SQLite catalog does not exist") if found != {holding[0]} and found: - raise refused(name, "its SQLite catalog's file system cannot be identified") + raise separation_refused( + name, "its SQLite catalog's file system cannot be identified" + ) if found & devices: - raise refused( + raise separation_refused( name, "its SQLite catalog is on a file system that holds a protected location, so " "a hard link there could reach a protected file", @@ -177,14 +176,14 @@ def own_device(name: str, folder: str, table: list[Mount] | None = None) -> int: try: status = os.stat(plain(name, folder)) except OSError: - raise refused(name, "its data location cannot be examined") from None + raise separation_refused(name, "its data location cannot be examined") from None if not stat.S_ISDIR(status.st_mode): - raise refused(name, "its data location is not a folder") + raise separation_refused(name, "its data location is not a folder") holding = _holding(Path(os.path.realpath(folder)), table) if holding is None or holding[2] not in LOCAL_FILE_SYSTEMS: - raise refused(name, f"its data location is on {_kind(holding)}{_LOCAL_ONLY}") + raise separation_refused(name, f"its data location is on {_kind(holding)}{_LOCAL_ONLY}") if holding[0] != status.st_dev: - raise refused( + raise separation_refused( name, "its data location's file system cannot be identified from the mount table" ) return status.st_dev @@ -216,10 +215,10 @@ def check_files( for path in paths: plain(name, path) if not path.startswith(prefix): - raise refused(name, "a snapshot references a file outside its data location") + raise separation_refused(name, "a snapshot references a file outside its data location") parts = path[len(prefix) :].split("/") if any(part in {"", ".", ".."} for part in parts): - raise refused(name, "a snapshot references a file by an ambiguous path") + raise separation_refused(name, "a snapshot references a file by an ambiguous path") relative.append(parts) if folder is None or not relative: return @@ -229,7 +228,7 @@ def check_files( opened[()] = root status = os.fstat(root) if (status.st_dev, status.st_ino) != folder: - raise refused(name, "its data folder was replaced after it was attached") + raise separation_refused(name, "its data folder was replaced after it was attached") for parts in relative: parent = root for depth in range(1, len(parts)): @@ -243,11 +242,17 @@ def check_files( finally: os.close(descriptor) if not stat.S_ISREG(status.st_mode): - raise refused(name, "a snapshot references something that is not a regular file") + raise separation_refused( + name, "a snapshot references something that is not a regular file" + ) if status.st_dev != folder[0]: - raise refused(name, "a snapshot references a file on another file system") + raise separation_refused( + name, "a snapshot references a file on another file system" + ) if (status.st_dev, status.st_ino) in protected: - raise refused(name, "a snapshot references a hard link to a protected file") + raise separation_refused( + name, "a snapshot references a hard link to a protected file" + ) finally: for descriptor in opened.values(): with contextlib.suppress(OSError): @@ -279,7 +284,7 @@ def _open(name: str, path: str, flags: int, folder: int | None) -> int: reason = "a snapshot references a file that does not exist" else: reason = f"a snapshot's file cannot be opened ({errno.errorcode.get(error.errno or 0)})" - raise refused(name, reason) from None + raise separation_refused(name, reason) from None def _is_link(path: str, folder: int | None) -> bool: @@ -297,7 +302,7 @@ def _mounts(name: str) -> list[Mount]: try: text = MOUNTINFO.read_text(encoding="utf-8", errors="replace") except OSError: - raise refused(name, "the mount table cannot be read on this platform") from None + raise separation_refused(name, "the mount table cannot be read on this platform") from None return parse_mounts(text) @@ -331,15 +336,17 @@ def _devices_under(name: str, location: str, table: list[Mount]) -> set[int]: break except FileNotFoundError: if existing.parent == existing: - raise refused(name, "a protected location cannot be examined") from None + raise separation_refused(name, "a protected location cannot be examined") from None existing = existing.parent except OSError: - raise refused(name, "a protected location cannot be examined") from None + raise separation_refused(name, "a protected location cannot be examined") from None below = [mount for mount in table if _within(mount[1], resolved)] holding = _holding(existing, table) for mount in [holding, *below]: if mount is None or mount[2] not in LOCAL_FILE_SYSTEMS: - raise refused(name, f"a protected location is on {_kind(mount)}{_LOCAL_ONLY}") + raise separation_refused( + name, f"a protected location is on {_kind(mount)}{_LOCAL_ONLY}" + ) return {device} | {mount[0] for mount in below} @@ -349,7 +356,7 @@ def _device_if_present(name: str, path: str) -> set[int]: except FileNotFoundError: return set() except OSError: - raise refused(name, "a protected file cannot be examined") from None + raise separation_refused(name, "a protected file cannot be examined") from None def _within(path: Path, folder: Path) -> bool: @@ -376,12 +383,14 @@ def check_credentials( is the boundary. `in_use` names every other variable of the instance, `values` their values where set. A lake without a credential of its own is refused.""" if in_use is None: - raise refused(name, "the instance's credentials in use were not given") + raise separation_refused(name, "the instance's credentials in use were not given") mine = set().union(*(credential_values(value) for value in own.values())) if not mine: - raise refused(name, "its object storage has no credential of its own") + raise separation_refused(name, "its object storage has no credential of its own") if set(own) & set(in_use): - raise refused(name, "its object-storage credential is another connection's variable") + raise separation_refused( + name, "its object-storage credential is another connection's variable" + ) for variable in in_use: if mine & credential_values(values.get(variable, "")): - raise refused(name, "its object-storage credential is another connection's") + raise separation_refused(name, "its object-storage credential is another connection's") diff --git a/packages/store/src/orvex/store/source_reader.py b/packages/store/src/orvex/store/source_reader.py index 58e9ff73..4b9321af 100644 --- a/packages/store/src/orvex/store/source_reader.py +++ b/packages/store/src/orvex/store/source_reader.py @@ -17,13 +17,13 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.errors import StoreError, register_secret, source_not_found from orvex.contracts.store.v1 import ( FileSourceAttachment, Parameter, PostgresSourceAttachment, SqliteSourceAttachment, ) -from orvex.store.errors import StoreError, _source_not_found, register_secret from orvex.store.locations import POSTGRES_CONNECT_TIMEOUT_S from orvex.store.nested_types import refuse_nested_uhugeint from orvex.store.parameters import _bound, _typed @@ -109,7 +109,7 @@ def _open( else: path = Path(source.path) if not path.is_file(): - raise _source_not_found(source) + raise source_not_found(source) _attach(source, connection, f"ATTACH {_quote(source.path)} AS {name} (READ_ONLY)") readable.append(path.as_posix()) connection.execute(f"USE {name}") diff --git a/packages/store/src/orvex/store/source_worker.py b/packages/store/src/orvex/store/source_worker.py index 080f42bc..81f9cd00 100644 --- a/packages/store/src/orvex/store/source_worker.py +++ b/packages/store/src/orvex/store/source_worker.py @@ -36,6 +36,7 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( FileSourceAttachment, Parameter, @@ -43,7 +44,6 @@ SqliteSourceAttachment, ) from orvex.store.credentials import _source_tls -from orvex.store.errors import StoreError from orvex.store.sql_text import literal as _quote DatabaseSource = FileSourceAttachment | SqliteSourceAttachment | PostgresSourceAttachment diff --git a/packages/store/src/orvex/store/sql_guard.py b/packages/store/src/orvex/store/sql_guard.py index 2ab57dc5..3987c36d 100644 --- a/packages/store/src/orvex/store/sql_guard.py +++ b/packages/store/src/orvex/store/sql_guard.py @@ -8,7 +8,7 @@ import itertools import duckdb -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError from orvex.store.file_names import _refuse_file_names from orvex.store.lake_names import METADATA_PREFIX from orvex.store.sql_parser import _plain, _tokens diff --git a/packages/store/src/orvex/store/sql_parser.py b/packages/store/src/orvex/store/sql_parser.py index ef9764af..568fc7ff 100644 --- a/packages/store/src/orvex/store/sql_parser.py +++ b/packages/store/src/orvex/store/sql_parser.py @@ -7,7 +7,7 @@ import threading import duckdb -from orvex.store.errors import StoreError +from orvex.contracts.store.errors import StoreError def _tokens(sql: str) -> list[tuple[int, duckdb.token_type]]: diff --git a/packages/store/src/orvex/store/sqlite_copy.py b/packages/store/src/orvex/store/sqlite_copy.py index f6ff4dd8..f6d4e635 100644 --- a/packages/store/src/orvex/store/sqlite_copy.py +++ b/packages/store/src/orvex/store/sqlite_copy.py @@ -7,8 +7,8 @@ import sqlite3 from pathlib import Path +from orvex.contracts.store.errors import StoreError, source_not_found from orvex.contracts.store.v1 import SqliteSourceAttachment -from orvex.store.errors import StoreError, _source_not_found def sqlite_backup(source: SqliteSourceAttachment, copy: Path) -> None: @@ -16,7 +16,7 @@ def sqlite_backup(source: SqliteSourceAttachment, copy: Path) -> None: pages only, no statement and no schema SQL is run on the original.""" path = Path(source.path) if not path.is_file(): - raise _source_not_found(source) + raise source_not_found(source) uri = path.absolute().as_uri() + "?mode=ro" try: original = sqlite3.connect(uri, uri=True) diff --git a/packages/store/src/orvex/store/transaction.py b/packages/store/src/orvex/store/transaction.py index ff9bb8a5..c053a3eb 100644 --- a/packages/store/src/orvex/store/transaction.py +++ b/packages/store/src/orvex/store/transaction.py @@ -14,11 +14,11 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import CommitResult, Parameter, Snapshot, StatementRequest from orvex.store import source_worker from orvex.store.catalog_values import _single_value from orvex.store.ctrl_c import _on_ctrl_c, _user_interrupt -from orvex.store.errors import StoreError from orvex.store.lake_names import LAKE, METADATA from orvex.store.limits import Guard from orvex.store.parameters import _bound, _typed diff --git a/pyproject.toml b/pyproject.toml index 21db1fed..95a466fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,8 @@ select = [ "tools/**/*.py" = ["T20"] # Conformance runners are command-line programs that report through stdout. "conformance/runners/*.py" = ["T20"] +# A module's own benchmarks of its internals are command-line programs that report through stdout. +"packages/*/benchmarks/*.py" = ["T20"] # The command line prints its user-facing output. "packages/server/src/*/server/cli.py" = ["T20"] # Generated contract types carry the contracts' descriptions verbatim. diff --git a/tools/benchmarks/query_baseline.py b/tools/benchmarks/query_baseline.py index ed7138dd..7d43b0a9 100644 --- a/tools/benchmarks/query_baseline.py +++ b/tools/benchmarks/query_baseline.py @@ -55,7 +55,7 @@ from orvex.server.identity import load_identity from orvex.server.instance import home_variable from orvex.server.project import load_project -from orvex.store.json_encoding import arrow_to_json +from orvex.server.store_access import store from tools.benchmarks import load_baseline @@ -125,7 +125,7 @@ def lake_tables(folder: Path) -> dict[str, dict[str, int]]: batch_size=1, ) ) - (row,) = json.loads("".join(arrow_to_json(counted)))["rows"] + (row,) = json.loads("".join(store().arrow_to_json(counted)))["rows"] found[name] = { "rows": int(row[0]), "bytes": sum(int(f.bytes or 0) for f in table.files), @@ -159,7 +159,7 @@ def reference_rows(folder: Path, sql: str) -> list[list[object]]: kind="read", sql=sql, parameters=[], snapshot=snapshot, batch_size=65_536 ) ) - rows: list[list[object]] = json.loads("".join(arrow_to_json(reader)))["rows"] + rows: list[list[object]] = json.loads("".join(store().arrow_to_json(reader)))["rows"] return rows finally: lake.close() diff --git a/tools/benchmarks/server_equivalence.py b/tools/benchmarks/server_equivalence.py index 6d56304d..97801df6 100644 --- a/tools/benchmarks/server_equivalence.py +++ b/tools/benchmarks/server_equivalence.py @@ -43,11 +43,11 @@ from pathlib import Path from typing import Any +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import ReadRequest from orvex.server import commands, lake_place from orvex.server.identity import load_identity from orvex.server.project import load_project -from orvex.store.lake import Lake SEED = 42 GIT_DATE = "2025-01-01T00:00:00+00:00" diff --git a/tools/benchmarks/source_stream.py b/tools/benchmarks/source_stream.py deleted file mode 100644 index 9cf27824..00000000 --- a/tools/benchmarks/source_stream.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Source isolation cost (#372 §5): landing one database-source table three ways. - - python -m tools.benchmarks.source_stream --rows 10000000 --rows 50000000 - -- `direct`: the source attached to the reading DuckDB instance and copied with - `CREATE TABLE … AS`, the path a lake used before #372 (no isolation); -- `stream`: a second DuckDB instance in the same process, its result streamed as Arrow - batches into the reading instance (in-process isolation, rejected by ADR-0031 §6); -- `worker`: the product's path, `source_worker.start`, an Arrow IPC stream from a worker - process (ADR-0031). - -Each path runs `--runs` times per size (default 3) on the same synthetic table (BIGINT, -DECIMAL(18,2), TIMESTAMPTZ, VARCHAR); the report gives the median wall time, the rows per -second and the peak resident memory of this (the reading) process. Every run checks the -landed row count and the exact sum of the decimal column against the source's. Nothing is -estimated: a failed check fails the benchmark. -""" - -from __future__ import annotations - -import argparse -import json -import platform -import resource -import statistics -import sys -import tempfile -import time -from collections.abc import Callable -from pathlib import Path - -import duckdb -from orvex.contracts.store.v1 import FileSourceAttachment -from orvex.store import source_worker - -MEMORY = "4GB" -THREADS = 4 -BATCH = 65_536 - - -def build(path: Path, rows: int) -> tuple[int, str]: - """The synthetic source table, and its row count and exact sum, computed by DuckDB.""" - with duckdb.connect(str(path)) as writer: - writer.sql( - "CREATE TABLE readings AS SELECT range AS id, " - "CAST((range % 100000) / 100 AS DECIMAL(18,2)) AS amount, " - "TIMESTAMPTZ '2026-01-01 00:00:00+00' + to_seconds(range % 31536000) AS at, " - f"'meter-' || CAST(range % 1000 AS VARCHAR) AS meter FROM range({int(rows)})" - ) - row = writer.execute( - "SELECT count(*), CAST(sum(amount) AS VARCHAR) FROM readings" - ).fetchone() - assert row is not None - return int(row[0]), str(row[1]) - - -def target() -> duckdb.DuckDBPyConnection: - return duckdb.connect(config={"memory_limit": MEMORY, "threads": THREADS}) - - -def check(connection: duckdb.DuckDBPyConnection, expected: tuple[int, str]) -> None: - row = connection.execute("SELECT count(*), CAST(sum(amount) AS VARCHAR) FROM landed").fetchone() - if row is None or (int(row[0]), str(row[1])) != expected: - raise SystemExit(f"wrong landing: {row} != {expected}") - - -def direct(path: Path, expected: tuple[int, str]) -> None: - with target() as connection: - connection.sql(f"ATTACH '{path}' AS source (READ_ONLY)") - connection.sql("CREATE TABLE landed AS SELECT * FROM source.readings") - check(connection, expected) - - -def stream(path: Path, expected: tuple[int, str]) -> None: - with duckdb.connect(config={"memory_limit": MEMORY, "threads": THREADS}) as source: - source.sql("SET arrow_lossless_conversion = true") - source.sql(f"ATTACH '{path}' AS source (READ_ONLY)") - reader = source.sql("SELECT * FROM source.readings").to_arrow_reader(BATCH) - with target() as connection: - connection.register("incoming", reader) - connection.sql("CREATE TABLE landed AS SELECT * FROM incoming") - check(connection, expected) - - -def worker(path: Path, expected: tuple[int, str]) -> None: - attachment = FileSourceAttachment( - kind="source_attachment", name="source", type="duckdb", path=str(path) - ) - started = source_worker.start( - attachment, - "", - "SELECT * FROM source.readings", - [], - memory_limit=MEMORY, - threads=THREADS, - timeout_ms=3_600_000, - batch_size=BATCH, - folder=path.parent, - ).open() - try: - with target() as connection: - connection.register("incoming", started.reader()) - connection.sql("CREATE TABLE landed AS SELECT * FROM incoming") - check(connection, expected) - finally: - started.close() - - -PATHS: dict[str, Callable[[Path, tuple[int, str]], None]] = { - "direct": direct, - "stream": stream, - "worker": worker, -} - - -def peak_mib() -> float: - """Peak resident memory of this process (the reading side), in MiB. A worker's own peak is - not reported: a spawned child's `ru_maxrss` counts the parent's pages from before `exec`.""" - scale = 1024 * 1024 if sys.platform == "darwin" else 1024 - return round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * scale / 2**20, 1) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--rows", type=int, action="append", required=True) - parser.add_argument("--runs", type=int, default=3) - arguments = parser.parse_args() - report: dict[str, object] = { - "python": platform.python_version(), - "duckdb": duckdb.__version__, - "machine": platform.platform(), - "cpus": len(__import__("os").sched_getaffinity(0)), - "memory_limit": MEMORY, - "threads": THREADS, - "batch_size": BATCH, - "results": [], - } - results: list[dict[str, object]] = [] - for rows in arguments.rows: - with tempfile.TemporaryDirectory(prefix="source-stream-") as folder: - path = Path(folder) / "source.duckdb" - expected = build(path, rows) - for name, land in PATHS.items(): - times = [] - for _ in range(arguments.runs): - started = time.perf_counter() - land(path, expected) - times.append(time.perf_counter() - started) - median = statistics.median(times) - own = peak_mib() - results.append( - { - "rows": rows, - "path": name, - "seconds": [round(t, 3) for t in times], - "median_s": round(median, 3), - "rows_per_s": round(rows / median), - "peak_rss_mib": own, - } - ) - print(json.dumps(results[-1]), flush=True) - report["results"] = results - print(json.dumps(report, indent=2)) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/benchmarks/store_baseline.py b/tools/benchmarks/store_baseline.py index 0bd81b71..f90d5848 100644 --- a/tools/benchmarks/store_baseline.py +++ b/tools/benchmarks/store_baseline.py @@ -10,8 +10,8 @@ process. The report gives the medians of `--runs` runs and the candidate's difference to the baseline. Operations, each on a fresh local lake: -- `open`: `open_lake` with no source; -- `attach`: `open_lake` with a Parquet, a CSV, a DuckDB and a SQLite source; +- `open`: `store().open_lake` with no source; +- `attach`: `store().open_lake` with a Parquet, a CSV, a DuckDB and a SQLite source; - `read`: a guarded aggregate read with typed parameters over 240,720 readings; - `transaction`: one transaction creating a table from a source, landing a DuckDB source, updating and deleting rows. @@ -33,6 +33,7 @@ from pathlib import Path from typing import Any +from orvex.contracts.store.client import Lake from orvex.contracts.store.v1 import ( FileSourceAttachment, LocalLakeLocation, @@ -41,7 +42,8 @@ SqliteSourceAttachment, StatementRequest, ) -from orvex.store.lake import Lake, Limits, open_lake +from orvex.contracts.store.values import Limits +from orvex.server.store_access import store OPERATIONS = ("open", "attach", "read", "transaction") LIMITS = Limits( @@ -74,14 +76,14 @@ def measure(operation: str, sources: list[Any], work: Path) -> dict[str, float]: started = time.perf_counter() if operation == "open": started = time.perf_counter() - lake = open_lake(location, LIMITS) + lake = store().open_lake(location, LIMITS) seconds = time.perf_counter() - started elif operation == "attach": started = time.perf_counter() - lake = open_lake(location, LIMITS, sources) + lake = store().open_lake(location, LIMITS, sources) seconds = time.perf_counter() - started elif operation == "read": - lake = open_lake(location, LIMITS, sources) + lake = store().open_lake(location, LIMITS, sources) with lake.transaction() as transaction: transaction.execute( _statement("CREATE TABLE readings AS SELECT * FROM readings_source.data") @@ -104,7 +106,7 @@ def measure(operation: str, sources: list[Any], work: Path) -> dict[str, float]: if rows.num_rows != 1: raise SystemExit("the aggregate read returned no row") elif operation == "transaction": - lake = open_lake(location, LIMITS, sources) + lake = store().open_lake(location, LIMITS, sources) started = time.perf_counter() with lake.transaction() as transaction: transaction.execute( diff --git a/tools/benchmarks/store_equivalence.py b/tools/benchmarks/store_equivalence.py index 9884224c..48de3510 100644 --- a/tools/benchmarks/store_equivalence.py +++ b/tools/benchmarks/store_equivalence.py @@ -45,6 +45,8 @@ import duckdb import pyarrow as pa +from orvex.contracts.store.client import Lake +from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( ChangesRequest, FileSourceAttachment, @@ -59,8 +61,8 @@ StatisticsRequest, TablesRequest, ) -from orvex.store.errors import StoreError -from orvex.store.lake import Lake, Limits, open_lake +from orvex.contracts.store.values import Limits +from orvex.server.store_access import store SEED = 42 LIMITS = Limits( @@ -587,14 +589,16 @@ def scenario(folder: Path, work: Path, out: Path) -> None: try: for name, location, files in locations(work, endpoint): observe = Observer() - lake = observe.step("open", partial(open_lake, location, LIMITS, sources(folder))) + lake = observe.step( + "open", partial(store().open_lake, location, LIMITS, sources(folder)) + ) if isinstance(lake, Lake): try: operations(observe, lake, folder, work) observe.step("state", partial(state, lake, files)) finally: lake.close() - reopened = observe.step("reopen", partial(open_lake, location, LIMITS)) + reopened = observe.step("reopen", partial(store().open_lake, location, LIMITS)) if isinstance(reopened, Lake): try: observe.step("state after reopen", partial(state, reopened, files)) @@ -604,7 +608,7 @@ def scenario(folder: Path, work: Path, out: Path) -> None: refusal = Observer() refusal.step( "a reserved source name", - lambda: open_lake( + lambda: store().open_lake( LocalLakeLocation( kind="lake_location", catalog="local", path=(work / "r").as_posix() ), @@ -621,7 +625,7 @@ def scenario(folder: Path, work: Path, out: Path) -> None: ) refusal.step( "limits missing", - lambda: open_lake( + lambda: store().open_lake( LocalLakeLocation( kind="lake_location", catalog="local", path=(work / "l").as_posix() ), diff --git a/tools/checks/materialization-allowlist.toml b/tools/checks/materialization-allowlist.toml index aeba0dce..b534acac 100644 --- a/tools/checks/materialization-allowlist.toml +++ b/tools/checks/materialization-allowlist.toml @@ -203,3 +203,13 @@ reason = "The access store (module state, not analytical data, #436): a user, a file = "packages/access/src/orvex/access/storage/sqlite.py" call = "fetchone" reason = "The access store's connection setup (#436): the one row of PRAGMA foreign_keys" + +[[allow]] +file = "packages/contracts/src/orvex/contracts/store/kit.py" +call = "to_pylist" +reason = "The store contract's conformance kit (#486 G1): the few rows of its own fixed cases, compared value by value to prove exactness; never a consumer's data" + +[[allow]] +file = "packages/contracts/src/orvex/contracts/store/kit_support.py" +call = "to_pylist" +reason = "The store contract's conformance kit (#486 G1): the few rows of its own fixed cases, read back to compare; never a consumer's data" diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index 5e77d54e..e3f4149e 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -80,6 +80,7 @@ regressions = [ "tests/properties/test_parser_fuzzing.py", "tests/properties/test_policy_broadening.py", "tests/properties/test_policy_properties.py", + "packages/store/tests/test_clickhouse_scanner_fuzzing.py", ] [[job]] @@ -754,3 +755,24 @@ setup = ["python -m pip install --quiet pip==26.3"] name = "standalone-contracts" run = "python -m tools.ci.standalone contracts" local = true + +[[job]] +workflow = "interfaces.yml" +job = "standalone-store" +kind = "check" +required = true +contexts = ["standalone-store"] +digest = "f3100f3e79efae0a" +setup = [ + 'dir="$RUNNER_TEMP/postgres-tls"', + "python -m pip install --quiet pip==26.3", + "python -m tools.ci.duckdb_extensions", + "for attempt in $(seq 60); do", +] +[[job.check]] +# The store installs and passes its own tests and its contract's kit alone, outside the +# repository, with the services its tests need (#486 §5 step 2, G4). +name = "standalone-store" +run = "python -m tools.ci.standalone store" +local = false +reason = "needs the PostgreSQL, ClickHouse and Azurite services the CI job starts" From bb9a22989ae2c2028657e2b895cb3a9f774beded Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 03:46:22 -0400 Subject: [PATCH 15/21] test(server): fault injections at the store binding reach no internals The time limit, the conflict and the other-writer faults of the run-record tests are injected through the store contract's binding (a helper's own calls pass the interception), and the canonical-bytes subprocess reaches the encoder through the store client. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- packages/server/tests/store_faults.py | 8 +++++--- packages/server/tests/test_run_records.py | 9 +++++---- tests/conformance/test_semantic_result.py | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/server/tests/store_faults.py b/packages/server/tests/store_faults.py index 8859093e..b483b727 100644 --- a/packages/server/tests/store_faults.py +++ b/packages/server/tests/store_faults.py @@ -20,17 +20,19 @@ @dataclasses.dataclass(frozen=True, slots=True) class Call: - """One intercepted call: the operation sent on, and the store to send others to.""" + """One intercepted call: the operation sent on, and the store to send others to, past the + interception (another writer's commit is not the intercepted one).""" bound: Bound envelope: CallEnvelope arguments: tuple[Any, ...] proceed: Callable[[], Any] + real: Callable[..., Any] def send(self, operation: str, *arguments: object) -> Any: # noqa: ANN401 - its result key = uuid.uuid4().hex if self.bound.contract.operation(operation).changes_state else None envelope = dataclasses.replace(self.envelope, idempotency_key=key) - return self.bound.call(operation, envelope, *arguments) + return self.real(self.bound, operation, envelope, *arguments) def another_writer(self, sql: str) -> None: """Another writer of the one open lake commits `sql` in a transaction of its own.""" @@ -60,7 +62,7 @@ def call(self: Bound, name: str, envelope: object, /, *arguments: Any, **keyword return real(self, name, envelope, *arguments, **keywords) assert isinstance(envelope, CallEnvelope) proceed = lambda: real(self, name, envelope, *arguments, **keywords) # noqa: E731 - return wrapper(Call(self, envelope, arguments, proceed)) + return wrapper(Call(self, envelope, arguments, proceed, real)) Bound.call = call # type: ignore[method-assign] return lambda: setattr(Bound, "call", real) diff --git a/packages/server/tests/test_run_records.py b/packages/server/tests/test_run_records.py index 9b6979ea..5617c2f5 100644 --- a/packages/server/tests/test_run_records.py +++ b/packages/server/tests/test_run_records.py @@ -853,10 +853,11 @@ def test_another_writer_first_fails_the_run_and_records_it_after_the_rollback( def timed_out(transaction: Transaction) -> StoreError: - """What the store's guard leaves when the transaction's time limit expires mid-statement.""" - return transaction._interrupted( - StoreError("STORE_TIMEOUT", "The transaction exceeded its time limit.", {}) - ) + """What the store leaves when the transaction's time limit expires mid-statement: the + statement's refusal, and a transaction that can no longer commit (the store's own guarantee, + #77), here through the contract's cancel.""" + transaction.cancel() + return StoreError("STORE_TIMEOUT", "The transaction exceeded its time limit.", {}) @pytest.mark.parametrize("where", ["count", "records"]) diff --git a/tests/conformance/test_semantic_result.py b/tests/conformance/test_semantic_result.py index 44e4fa4b..13e439ce 100644 --- a/tests/conformance/test_semantic_result.py +++ b/tests/conformance/test_semantic_result.py @@ -186,6 +186,7 @@ def test_the_canonical_form_keeps_every_value_exact() -> None: sys.path.insert(0, {root!r}) from tests.conformance.test_semantic_result import detail, exact_table, EXACT_COLUMNS from tests.support.semantic_results import top_n_with_others +from orvex.server.store_access import store parts = [ top_n_with_others().canonical(store().arrow_to_json), detail(exact_table(), EXACT_COLUMNS).canonical(store().arrow_to_json), From d40bdb23f9cad2ab640985177090e27becf737c0 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 07:55:58 -0400 Subject: [PATCH 16/21] refactor(store): Arrow crosses the store contract as the foundations' Arrow table type The store contract's reads, loads and JSON encoding carry `pyarrow.Table` values, encoded by the foundations' codec; the store's own IPC module and its codec changes are dropped, and the standalone store job is optional like the other interface jobs until main requires them. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .../src/orvex/contracts/store/arrow_ipc.py | 37 -------------- .../src/orvex/contracts/store/client.py | 35 +++++++------ .../src/orvex/contracts/store/interface.py | 15 +++--- .../src/orvex/contracts/store/kit.py | 15 +++--- .../src/orvex/contracts/store/kit_support.py | 5 +- .../src/orvex/contracts/store/values.py | 7 +-- packages/contracts/tests/test_codec_values.py | 50 ------------------- .../contracts/tests/test_store_arrow_ipc.py | 29 ----------- packages/store/src/orvex/store/api/service.py | 28 +++++------ packages/store/tests/test_contract_kit.py | 10 ++-- tools/ci/checks.toml | 2 +- 11 files changed, 57 insertions(+), 176 deletions(-) delete mode 100644 packages/contracts/src/orvex/contracts/store/arrow_ipc.py delete mode 100644 packages/contracts/tests/test_codec_values.py delete mode 100644 packages/contracts/tests/test_store_arrow_ipc.py diff --git a/packages/contracts/src/orvex/contracts/store/arrow_ipc.py b/packages/contracts/src/orvex/contracts/store/arrow_ipc.py deleted file mode 100644 index d0491099..00000000 --- a/packages/contracts/src/orvex/contracts/store/arrow_ipc.py +++ /dev/null @@ -1,37 +0,0 @@ -"""How Arrow crosses the store's interface contract, on both sides: as IPC streams, exactly -(every type, its metadata and every NULL as written), never as a live reader.""" - -from __future__ import annotations - -from collections.abc import Iterable - -import pyarrow as pa - - -def stream_bytes(schema: pa.Schema, batches: Iterable[pa.RecordBatch] = ()) -> bytes: - """`batches` as one IPC stream of `schema` (dictionaries included); no batch: the schema.""" - sink = pa.BufferOutputStream() - with pa.ipc.new_stream(sink, schema) as writer: - for batch in batches: - writer.write_batch(batch) - written: bytes = sink.getvalue().to_pybytes() - return written - - -def batch_bytes(batch: pa.RecordBatch, schema: pa.Schema | None = None) -> bytes: - """One batch as an IPC stream of its own, written with `schema` (its own by default).""" - return stream_bytes(batch.schema if schema is None else schema, (batch,)) - - -def stream_reader(data: bytes) -> pa.RecordBatchReader: - """The reader of one IPC stream: its batches are decoded as they are read.""" - return pa.ipc.open_stream(pa.py_buffer(data)) - - -def stream_schema(data: bytes) -> pa.Schema: - return stream_reader(data).schema - - -def one_batch(data: bytes) -> pa.RecordBatch: - """The one batch of a stream written by `batch_bytes`.""" - return stream_reader(data).read_next_batch() diff --git a/packages/contracts/src/orvex/contracts/store/client.py b/packages/contracts/src/orvex/contracts/store/client.py index 558554aa..5ffc9351 100644 --- a/packages/contracts/src/orvex/contracts/store/client.py +++ b/packages/contracts/src/orvex/contracts/store/client.py @@ -19,7 +19,6 @@ import pyarrow as pa from orvex.contracts.binding import Bound from orvex.contracts.envelope import CallEnvelope -from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch, stream_bytes, stream_schema from orvex.contracts.store.errors import StoreError from orvex.contracts.store.interface import STORE, Location, Source from orvex.contracts.store.v1 import ( @@ -96,17 +95,18 @@ def open_lake( def reader(self, ticket: ReadTicket) -> pa.RecordBatchReader: """The Arrow stream of a running read; the read is let go when it ends or is dropped.""" - schema = stream_schema(ticket.schema) + schema = ticket.schema.schema return pa.RecordBatchReader.from_batches(schema, _Batches(self, ticket)) def arrow_to_json(self, reader: pa.RecordBatchReader) -> Iterator[str]: """The `ResultJson` document of `reader` as text fragments, one batch at a time; unsupported types are reported before any row is produced.""" - frame: JsonFrame = self.call("json_frame", stream_bytes(reader.schema)) + frame: JsonFrame = self.call("json_frame", reader.schema.empty_table()) yield from frame.head first = True for batch in reader: - fragments = self.call("json_rows", batch_bytes(batch, reader.schema), first) + rows = pa.Table.from_batches([batch], reader.schema) + fragments = self.call("json_rows", rows, first) first = first and not fragments yield from fragments yield frame.tail @@ -142,22 +142,25 @@ def __init__(self, client: StoreClient, ticket: ReadTicket) -> None: self._client = client self._ticket = ticket self._open = True + self._pending: list[pa.RecordBatch] = [] def __iter__(self) -> _Batches: return self def __next__(self) -> pa.RecordBatch: - if not self._open: - raise StopIteration - try: - data = self._client.call("read_batch", self._ticket) - except BaseException: - self.close() - raise - if data is None: - self.close() - raise StopIteration - return one_batch(data) + while not self._pending: + if not self._open: + raise StopIteration + try: + data = self._client.call("read_batch", self._ticket) + except BaseException: + self.close() + raise + if data is None: + self.close() + raise StopIteration + self._pending = data.to_batches() + return self._pending.pop(0) def close(self) -> None: if self._open: @@ -305,7 +308,7 @@ def load_arrow( timeout_ms: int | None = None, ) -> int: """Create `schema.table` from an Arrow stream, each column cast to its DuckDB type.""" - data = stream_bytes(reader.schema, reader) + data = reader.read_all() found: int = self._client.call( "load_arrow", self.handle, schema, table, data, tuple(columns), timeout_ms ) diff --git a/packages/contracts/src/orvex/contracts/store/interface.py b/packages/contracts/src/orvex/contracts/store/interface.py index d976c6e2..f1f31326 100644 --- a/packages/contracts/src/orvex/contracts/store/interface.py +++ b/packages/contracts/src/orvex/contracts/store/interface.py @@ -1,9 +1,9 @@ """The store's interface contract (#486 §5 step 2, ADR-0033): every operation its consumers use, by capability, with the codes each may raise (`orvex.contracts.store.errors.StoreError`). -Each operation takes the call envelope first. Arrow crosses as IPC streams (`bytes`): a read's -schema in its handle and then one batch per `read_batch`, never a live reader. What the store -holds open (lakes, transactions, reads, connections) is named by a handle +Each operation takes the call envelope first. Arrow crosses as tables (the codec's Arrow type): a +read's schema in its handle and then one batch per `read_batch`, never a live reader. What the +store holds open (lakes, transactions, reads, connections) is named by a handle (`orvex.contracts.store.values`); the consumer's facade is `orvex.contracts.store.client`. An operation changes state when it changes a lake, or what later calls see of a lake, a @@ -18,6 +18,7 @@ from datetime import datetime from typing import Protocol +import pyarrow as pa from orvex.contracts.envelope import CallEnvelope from orvex.contracts.interface import InterfaceContract, Operation from orvex.contracts.store.errors import StoreError @@ -121,7 +122,7 @@ def start_read( self, envelope: CallEnvelope, lake: LakeHandle, request: ReadRequest ) -> ReadHandle: ... - def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: ... + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> pa.Table | None: ... def cancel_read(self, envelope: CallEnvelope, read: ReadHandle) -> None: ... @@ -156,7 +157,7 @@ def load_arrow( transaction: TransactionHandle, schema: str, table: str, - data: bytes, + data: pa.Table, columns: Columns, timeout_ms: int | None, ) -> int: ... @@ -254,9 +255,9 @@ def start_clickhouse_read( ) -> ReadHandle: ... # --- encoding ------------------------------------------------------------------------------ - def json_frame(self, envelope: CallEnvelope, schema: bytes) -> JsonFrame: ... + def json_frame(self, envelope: CallEnvelope, schema: pa.Table) -> JsonFrame: ... - def json_rows(self, envelope: CallEnvelope, batch: bytes, first: bool) -> tuple[str, ...]: ... + def json_rows(self, envelope: CallEnvelope, rows: pa.Table, first: bool) -> tuple[str, ...]: ... # --- separation ---------------------------------------------------------------------------- def is_object_storage(self, envelope: CallEnvelope, location: str) -> bool: ... diff --git a/packages/contracts/src/orvex/contracts/store/kit.py b/packages/contracts/src/orvex/contracts/store/kit.py index 162dd6a7..ca306b57 100644 --- a/packages/contracts/src/orvex/contracts/store/kit.py +++ b/packages/contracts/src/orvex/contracts/store/kit.py @@ -18,7 +18,6 @@ from orvex.contracts.envelope import CallEnvelope from orvex.contracts.kit import Idempotent, Kit, KitCase from orvex.contracts.kit_client import KitClient -from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch, stream_bytes, stream_schema from orvex.contracts.store.interface import STORE from orvex.contracts.store.kit_support import ( EXISTING_LIMITS, @@ -60,7 +59,7 @@ "TIMESTAMPTZ '2026-03-29 01:30:00+00' AS at, NULL::BIGINT AS missing" ) FORGED_LAKE = LakeHandle("lake-forged") -FORGED_READ = ReadHandle("read-forged", b"") +FORGED_READ = ReadHandle("read-forged", pa.table({})) FORGED_TRANSACTION = TransactionHandle("transaction-forged", None) FORGED_EXISTING = ExistingLakeHandle("existing-lake-forged", "forged") FORGED_ENGINE = ClickHouseHandle("clickhouse-forged", "db", "") @@ -237,7 +236,7 @@ def _read_ends(client: KitClient) -> None: read = client.start_read(lake, read_request("SELECT a FROM t ORDER BY a", snapshot)) seen = [] while (data := client.read_batch(read)) is not None: - seen += one_batch(data).column("a").to_pylist() + seen += data.column("a").to_pylist() check(seen == [0, 1, 2], f"every row, once: {seen}") check(client.read_batch(read) is None, "an ended read stays ended") client.close_read(read) @@ -331,7 +330,7 @@ def _load_exact(client: KitClient) -> None: transaction, "main", "loaded", - batch_bytes(batch), + pa.Table.from_batches([batch]), (("amount", "DECIMAL(18,2)"),), None, ) @@ -506,7 +505,7 @@ def run(client: KitClient) -> None: # --- encoding ------------------------------------------------------------------------------------ -def _read_schema(client: KitClient) -> tuple[bytes, pa.RecordBatch]: +def _read_schema(client: KitClient) -> tuple[pa.Table, pa.RecordBatch]: lake = open_lake(client) write(client, lake, "CREATE TABLE t AS SELECT 1.50::DECIMAL(18,2) AS a, NULL::HUGEINT AS h") read = client.start_read( @@ -526,13 +525,13 @@ def _json_frame(client: KitClient) -> None: '{"name": "h", "type": "HUGEINT"}],"rows":[' ) check(found == expected, f"the head: {found}") - bare = stream_bytes(pa.schema([("x", pa.int64())])) + bare = pa.schema([("x", pa.int64())]).empty_table() refused("STORE_TYPE_UNSUPPORTED", lambda: client.json_frame(bare)) def _json_rows(client: KitClient) -> None: schema, batch = _read_schema(client) - stream = batch_bytes(batch, stream_schema(schema)) + stream = pa.Table.from_batches([batch], schema.schema) check(client.json_rows(stream, True) == ('["1.50", null]',), "exact text, NULL as null") check(client.json_rows(stream, False) == (',["1.50", null]',), "a later row is separated") @@ -751,7 +750,7 @@ def _case( _loading.of(c), "main", "loaded", - batch_bytes(pa.record_batch({"a": pa.array([1, 2], pa.int64())})), + pa.table({"a": pa.array([1, 2], pa.int64())}), (("a", "BIGINT"),), None, ), diff --git a/packages/contracts/src/orvex/contracts/store/kit_support.py b/packages/contracts/src/orvex/contracts/store/kit_support.py index fd2bb1e2..e04979b6 100644 --- a/packages/contracts/src/orvex/contracts/store/kit_support.py +++ b/packages/contracts/src/orvex/contracts/store/kit_support.py @@ -14,7 +14,6 @@ import pyarrow as pa from orvex.contracts.kit_client import KitClient -from orvex.contracts.store.arrow_ipc import one_batch, stream_schema from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( CommitResult, @@ -100,11 +99,11 @@ def commit(client: KitClient, lake: LakeHandle, *sqls: str) -> Snapshot: def drain(client: KitClient, read: ReadHandle) -> pa.Table: """Every batch of a read, then the read let go.""" - schema = stream_schema(read.schema) + schema = read.schema.schema batches = [] try: while (data := client.read_batch(read)) is not None: - batches.append(one_batch(data)) + batches += data.to_batches() finally: client.close_read(read) return pa.Table.from_batches(batches, schema=schema) diff --git a/packages/contracts/src/orvex/contracts/store/values.py b/packages/contracts/src/orvex/contracts/store/values.py index a5cde37f..366d9194 100644 --- a/packages/contracts/src/orvex/contracts/store/values.py +++ b/packages/contracts/src/orvex/contracts/store/values.py @@ -11,6 +11,7 @@ from dataclasses import dataclass +import pyarrow as pa from orvex.contracts.store.v1 import Snapshot, StoreError @@ -101,11 +102,11 @@ class TransactionHandle: @dataclass(frozen=True, slots=True) class ReadHandle: - """A running read: its Arrow schema as an IPC stream with no batch, then its batches one at a - time (`read_batch`).""" + """A running read: its Arrow schema, as a table with no row, then its batches one at a time + (`read_batch`).""" id: str - schema: bytes + schema: pa.Table @dataclass(frozen=True, slots=True) diff --git a/packages/contracts/tests/test_codec_values.py b/packages/contracts/tests/test_codec_values.py deleted file mode 100644 index 878275d5..00000000 --- a/packages/contracts/tests/test_codec_values.py +++ /dev/null @@ -1,50 +0,0 @@ -"""#486 G5: values the codec must carry exactly that the store contract showed it refused: a model -with field aliases, and an instant whose fixed offset is pydantic's own zone class.""" - -from __future__ import annotations - -import datetime as dt -from zoneinfo import ZoneInfo - -import pytest -from orvex.contracts.codec import Codec -from orvex.contracts.interface_errors import InterfaceContractError -from orvex.contracts.store.v1 import LakeTable, Snapshot, TableColumn - - -def test_a_model_with_an_aliased_field_crosses_exactly() -> None: - column = TableColumn(name="a", type="DECIMAL(18,2)", nullable=False) - table = LakeTable(schema="main", table="t", columns=[column]) - codec = Codec(frozenset({LakeTable})) - assert codec.decode(codec.encode(table)) == table - - -def test_an_instant_parsed_by_pydantic_keeps_its_offset() -> None: - snapshot = Snapshot.model_validate( - {"kind": "snapshot", "id": 1, "committed_at": "2026-03-29T01:30:00+02:00"} - ) - assert type(snapshot.committed_at.tzinfo) is not dt.timezone # pydantic's own class - codec = Codec(frozenset({Snapshot})) - crossed = codec.decode(codec.encode(snapshot.committed_at)) - assert isinstance(crossed, dt.datetime) - assert crossed == snapshot.committed_at - assert crossed.utcoffset() == dt.timedelta(hours=2) - - -class _Shifting(dt.tzinfo): - """A zone whose offset depends on the instant: not a fixed offset, and not an IANA key.""" - - def utcoffset(self, moment: dt.datetime | None) -> dt.timedelta: - return dt.timedelta(hours=1 if moment is None or moment.month < 6 else 2) - - def dst(self, moment: dt.datetime | None) -> dt.timedelta: - return dt.timedelta(0) - - -def test_a_zone_that_is_neither_fixed_nor_named_is_refused() -> None: - codec = Codec(frozenset()) - with pytest.raises(InterfaceContractError) as caught: - codec.encode(dt.datetime(2026, 7, 1, tzinfo=_Shifting())) - assert caught.value.code == "INTERFACE_VALUE_NOT_SERIALIZABLE" - named = dt.datetime(2026, 7, 1, tzinfo=ZoneInfo("America/Lima")) - assert codec.decode(codec.encode(named)) == named diff --git a/packages/contracts/tests/test_store_arrow_ipc.py b/packages/contracts/tests/test_store_arrow_ipc.py deleted file mode 100644 index 0c848033..00000000 --- a/packages/contracts/tests/test_store_arrow_ipc.py +++ /dev/null @@ -1,29 +0,0 @@ -"""#486 G5: Arrow crosses the store's contract as IPC streams, exactly and batch by batch.""" - -from __future__ import annotations - -from decimal import Decimal - -import pyarrow as pa -from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch, stream_bytes, stream_reader - - -def test_ipc_carries_arrow_exactly_and_lazily() -> None: - metadata = {b"duckdb_type": b"DECIMAL(38,10)"} - field = pa.field("amount", pa.decimal128(38, 10), metadata=metadata) - batch = pa.record_batch( - [pa.array([Decimal("12345678901234567890.1234567890"), None], pa.decimal128(38, 10))], - schema=pa.schema([field]), - ) - assert stream_reader(stream_bytes(batch.schema)).schema == batch.schema - crossed = one_batch(batch_bytes(batch)) - assert crossed.equals(batch) - assert crossed.schema.equals(batch.schema, check_metadata=True) - reader = stream_reader(batch_bytes(batch)) - assert isinstance(reader, pa.RecordBatchReader) # a stream: batches are read on demand - assert reader.read_next_batch().equals(batch) - - -def test_a_dictionary_column_crosses_in_its_own_stream() -> None: - batch = pa.record_batch({"zone": pa.array(["north", "south", "north"]).dictionary_encode()}) - assert one_batch(batch_bytes(batch)).equals(batch) diff --git a/packages/store/src/orvex/store/api/service.py b/packages/store/src/orvex/store/api/service.py index 69874945..81f6cbf1 100644 --- a/packages/store/src/orvex/store/api/service.py +++ b/packages/store/src/orvex/store/api/service.py @@ -11,13 +11,6 @@ import pyarrow as pa from orvex.contracts.envelope import CallEnvelope -from orvex.contracts.store.arrow_ipc import ( - batch_bytes, - one_batch, - stream_bytes, - stream_reader, - stream_schema, -) from orvex.contracts.store.errors import StoreError from orvex.contracts.store.v1 import ( ChangesRequest, @@ -183,7 +176,7 @@ def start_read( running = self._lake(lake).start_read(request) return self._reading(running.reader, running.cancel, lake.id) - def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> pa.Table | None: entry = self._held.entry(READ, read.id) held = entry.value assert isinstance(held, _Read) @@ -193,7 +186,7 @@ def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: batch = held.reader.read_next_batch() except StopIteration: return None - return batch_bytes(batch) + return pa.Table.from_batches([batch]) def cancel_read(self, envelope: CallEnvelope, read: ReadHandle) -> None: def work() -> None: @@ -264,7 +257,7 @@ def load_arrow( transaction: TransactionHandle, schema: str, table: str, - data: bytes, + data: pa.Table, columns: tuple[tuple[str, str], ...], timeout_ms: int | None, ) -> int: @@ -274,7 +267,7 @@ def load_arrow( "load_arrow", envelope.idempotency_key, lambda: open_.load_arrow( - schema, table, stream_reader(data), columns, timeout_ms=timeout_ms + schema, table, data.to_reader(), columns, timeout_ms=timeout_ms ), ) @@ -441,11 +434,14 @@ def start_clickhouse_read( # --- encoding ------------------------------------------------------------------------------ - def json_frame(self, envelope: CallEnvelope, schema: bytes) -> JsonFrame: - return JsonFrame(tuple(json_head(stream_schema(schema))), JSON_TAIL) + def json_frame(self, envelope: CallEnvelope, schema: pa.Table) -> JsonFrame: + return JsonFrame(tuple(json_head(schema.schema)), JSON_TAIL) - def json_rows(self, envelope: CallEnvelope, batch: bytes, first: bool) -> tuple[str, ...]: - return tuple(json_rows(one_batch(batch), first)) + def json_rows(self, envelope: CallEnvelope, rows: pa.Table, first: bool) -> tuple[str, ...]: + fragments: list[str] = [] + for batch in rows.to_batches(): + fragments += json_rows(batch, first and not fragments) + return tuple(fragments) # --- separation ---------------------------------------------------------------------------- @@ -496,5 +492,5 @@ def _clickhouse(self, engine: ClickHouseHandle) -> ClickHouse: def _reading( self, reader: pa.RecordBatchReader, cancel: Callable[[], None] | None, owner: str ) -> ReadHandle: - schema = stream_bytes(reader.schema) + schema = reader.schema.empty_table() return ReadHandle(self._held.add(READ, _Read(reader, cancel), owner), schema) diff --git a/packages/store/tests/test_contract_kit.py b/packages/store/tests/test_contract_kit.py index a4e60e09..a0389c84 100644 --- a/packages/store/tests/test_contract_kit.py +++ b/packages/store/tests/test_contract_kit.py @@ -10,7 +10,6 @@ from orvex.contracts.envelope import CallEnvelope from orvex.contracts.kit_run import kit_problems, run_kit, verify_kit from orvex.contracts.manifest import parse_manifest -from orvex.contracts.store.arrow_ipc import batch_bytes, one_batch from orvex.contracts.store.interface import STORE from orvex.contracts.store.kit import STORE_KIT from orvex.contracts.store.v1 import StatementRequest @@ -52,22 +51,21 @@ def execute( class _DecimalAsFloat(Store): """Planted: a read that turns every DECIMAL into a float.""" - def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> pa.Table | None: found = super().read_batch(envelope, read) if found is None: return None - batch = one_batch(found) columns = [ column.cast(pa.float64()) if pa.types.is_decimal(column.type) else column - for column in batch.columns + for column in found.columns ] - return batch_bytes(pa.record_batch(columns, names=batch.schema.names)) + return pa.table(columns, names=found.column_names) class _ForgedHandleAccepted(Store): """Planted: a forged read handle answered as an ended read instead of refused.""" - def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> bytes | None: + def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> pa.Table | None: if read.id == "read-forged": return None return super().read_batch(envelope, read) diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index e3f4149e..1357c436 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -760,7 +760,7 @@ local = true workflow = "interfaces.yml" job = "standalone-store" kind = "check" -required = true +required = false contexts = ["standalone-store"] digest = "f3100f3e79efae0a" setup = [ From 396b2aaa8628086e5e1367710f760b43a5733286 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:12:17 -0400 Subject: [PATCH 17/21] build(store): the store's tests declare orvex-testing, and its contract's files are counted Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- docs/process/git-workflow.md | 7 +++++++ packages/server/tests/test_lake_data.py | 1 - packages/store/src/orvex/store/module.toml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/process/git-workflow.md b/docs/process/git-workflow.md index 7463f89c..b709fb21 100644 --- a/docs/process/git-workflow.md +++ b/docs/process/git-workflow.md @@ -131,6 +131,13 @@ packages/contracts/src/orvex/contracts/ generated contract types !packages/contracts/src/orvex/contracts/kit_client.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/kit_process.py hand-written: counted (#486, interface contracts) !packages/contracts/src/orvex/contracts/version.py hand-written: counted (#486, interface contracts) +!packages/contracts/src/orvex/contracts/store/client.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/client_sources.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/errors.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/interface.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/kit.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/kit_support.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/values.py hand-written: counted (#486, the store contract) tests/contracts/generated/ generated contract fixtures !tests/contracts/generated/__init__.py hand-written: counted web/src/lib/contracts/ generated TypeScript types diff --git a/packages/server/tests/test_lake_data.py b/packages/server/tests/test_lake_data.py index c2a3d52f..4d2bf222 100644 --- a/packages/server/tests/test_lake_data.py +++ b/packages/server/tests/test_lake_data.py @@ -25,7 +25,6 @@ from orvex.server.identity import load_identity from orvex.server.project import load_project from orvex.server.store_access import store - from orvex.testing import azurite SLUG = load_identity().slug diff --git a/packages/store/src/orvex/store/module.toml b/packages/store/src/orvex/store/module.toml index 8c296210..0cd22eed 100644 --- a/packages/store/src/orvex/store/module.toml +++ b/packages/store/src/orvex/store/module.toml @@ -4,7 +4,7 @@ module = "store" [dependencies] runtime = ["duckdb", "orvex-contracts", "pyarrow"] -tests = ["hypothesis", "moto", "pytest"] +tests = ["hypothesis", "moto", "orvex-testing", "pytest"] [[provides]] contract = "store" From 1922be5e9881d43a09873dbd249c2c330cfe8437 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:12:17 -0400 Subject: [PATCH 18/21] refactor(checks): tests, runners and tools reach the server only through its composition root The consumer rule allows `server.assembly`, and nothing else of the server, outside the server. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- tests/tools/test_consumer_boundaries.py | 15 ++++++++++++++- tools/checks/consumers.py | 11 +++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/tools/test_consumer_boundaries.py b/tests/tools/test_consumer_boundaries.py index cf365809..2681bb75 100644 --- a/tests/tools/test_consumer_boundaries.py +++ b/tests/tools/test_consumer_boundaries.py @@ -31,6 +31,10 @@ "internal in assembly": (ASSEMBLY, "from orvex.store.lake import x\n"), "test helpers in a runner": ("conformance/runners/y.py", "import orvex.testing.environment\n"), "test helpers in a tool": ("tools/z.py", "from orvex.testing.fixtures import frozen\n"), + "a server internal beside the assembly": ( + "tests/x/test_w.py", + "from orvex.server.assembly import provider\nfrom orvex.server import cli\n", + ), } @@ -66,6 +70,15 @@ def test_a_planted_internal_import_is_caught(relative: str, source: str) -> None "packages/server/tests/t.py", "from orvex.testing import azurite\n", ), + "the composition root from a test": ( + "tests/x/t.py", + "from orvex.server.assembly import provider\n", + ), + "the composition root from a runner": ( + "conformance/runners/x.py", + "import orvex.server.assembly\n", + ), + "the composition root from a tool": ("tools/x.py", "from orvex.server import assembly\n"), } @@ -93,7 +106,7 @@ def test_the_scan_covers_the_consumers_and_leaves_the_modules_to_the_module_rule assert "server 4" in summary assert "runners 2" in summary assert "tools 3" in summary - assert "tests 2" in summary + assert "tests 3" in summary def test_report_mode_prints_every_violation_and_does_not_fail( diff --git a/tools/checks/consumers.py b/tools/checks/consumers.py index 272e5653..a4a40831 100644 --- a/tools/checks/consumers.py +++ b/tools/checks/consumers.py @@ -1,7 +1,8 @@ """Consumer boundaries (#486 G2): `server`, the conformance runners, `tools/` and the repository -tests import only `.contracts`; `server/assembly.py` may also import the modules' -`api` packages, and the server's own files the server. Tests (`tests/`, a package's `tests/`) -may also import the shared test helpers, `.testing`; nothing else may. +tests import only `.contracts` and, from the server, its composition root +`server.assembly`; `server/assembly.py` may also import the modules' `api` packages, and the +server's own files the server. Tests (`tests/`, a package's `tests/`) may also import the shared +test helpers, `.testing`; nothing else may. CONSUMER_BOUNDARY_VIOLATION: any other import of a module, as `tools/checks/imports.py` reads it (static, or dynamic with a constant name). In report mode until #486 step 7: @@ -29,6 +30,8 @@ } SHARED = "contracts" PUBLIC = "api" +COMPOSER = "server" +ROOT = "assembly" # the composition root, the server's one face to the other consumers def assembly(namespace: str) -> str: @@ -58,7 +61,7 @@ def consumer_findings(tree: ast.AST, namespace: str, relative: str) -> list[Find allowed = ( target == SHARED or (target == TESTING and _is_test(relative)) - or (area == "server" and target == "server") + or (target == COMPOSER and (area == "server" or parts[2:3] in ([], [ROOT]))) or (relative == assembly(namespace) and len(parts) > 2 and parts[2] == PUBLIC) ) if allowed or (target, line) in reported: From 8e7f0efdd926a7b655cf40ca81f7e738b0da05ba Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 08:49:01 -0400 Subject: [PATCH 19/21] feat(store): each operation declares only the codes it raises, and the kit proves every one The store contract's operations declare their own error codes; `kit_codes` holds one case per (operation, code), each making that operation refuse with that code, and the kit is refused when a declared code has no case or a case proves an undeclared code (KIT_CODE_UNPROVEN). The ClickHouse cases take a server from the environment and fail without one (`kit_services`); refusal-only cases the code cases now prove are removed from the kit (153 cases). Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- docs/process/git-workflow.md | 2 + packages/contracts/interface-record.json | 2083 +++++++---------- .../src/orvex/contracts/store/interface.py | 92 +- .../src/orvex/contracts/store/kit.py | 202 +- .../src/orvex/contracts/store/kit_codes.py | 756 ++++++ .../src/orvex/contracts/store/kit_services.py | 77 + .../contracts/tests/test_store_kit_codes.py | 46 + packages/store/tests/test_contract_kit.py | 2 +- relative/x/catalog.ducklake | Bin 0 -> 1847296 bytes 9 files changed, 1798 insertions(+), 1462 deletions(-) create mode 100644 packages/contracts/src/orvex/contracts/store/kit_codes.py create mode 100644 packages/contracts/src/orvex/contracts/store/kit_services.py create mode 100644 packages/contracts/tests/test_store_kit_codes.py create mode 100644 relative/x/catalog.ducklake diff --git a/docs/process/git-workflow.md b/docs/process/git-workflow.md index b709fb21..0cb8896d 100644 --- a/docs/process/git-workflow.md +++ b/docs/process/git-workflow.md @@ -137,6 +137,8 @@ packages/contracts/src/orvex/contracts/ generated contract types !packages/contracts/src/orvex/contracts/store/interface.py hand-written: counted (#486, the store contract) !packages/contracts/src/orvex/contracts/store/kit.py hand-written: counted (#486, the store contract) !packages/contracts/src/orvex/contracts/store/kit_support.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/kit_codes.py hand-written: counted (#486, the store contract) +!packages/contracts/src/orvex/contracts/store/kit_services.py hand-written: counted (#486, the store contract) !packages/contracts/src/orvex/contracts/store/values.py hand-written: counted (#486, the store contract) tests/contracts/generated/ generated contract fixtures !tests/contracts/generated/__init__.py hand-written: counted diff --git a/packages/contracts/interface-record.json b/packages/contracts/interface-record.json index 13ba6cae..d17e5366 100644 --- a/packages/contracts/interface-record.json +++ b/packages/contracts/interface-record.json @@ -46,24 +46,90 @@ "operation": "begin", "since": "1.0" }, + "begin.raises.store-busy": { + "expectation": "begin refuses with STORE_BUSY", + "idempotency": false, + "operation": "begin", + "since": "1.0" + }, + "begin.raises.store-parameter-invalid": { + "expectation": "begin refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "begin", + "since": "1.0" + }, + "begin.raises.store-statement-invalid": { + "expectation": "begin refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "begin", + "since": "1.0" + }, + "begin.raises.store-timeout": { + "expectation": "begin refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "begin", + "since": "1.0" + }, "cancel-read.idempotent": { "expectation": "one key cancels once", "idempotency": true, "operation": "cancel_read", "since": "1.0" }, + "cancel-read.raises.store-parameter-invalid": { + "expectation": "cancel_read refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "cancel_read", + "since": "1.0" + }, "cancel-transaction.idempotent": { "expectation": "one key cancels once", "idempotency": true, "operation": "cancel_transaction", "since": "1.0" }, + "cancel-transaction.raises.store-parameter-invalid": { + "expectation": "cancel_transaction refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "cancel_transaction", + "since": "1.0" + }, "cancel-transaction.rolls-back": { "expectation": "a cancelled transaction never commits", "idempotency": false, "operation": "cancel_transaction", "since": "1.0" }, + "changes.raises.store-busy": { + "expectation": "changes refuses with STORE_BUSY", + "idempotency": false, + "operation": "changes", + "since": "1.0" + }, + "changes.raises.store-parameter-invalid": { + "expectation": "changes refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "changes", + "since": "1.0" + }, + "changes.raises.store-snapshot-expired": { + "expectation": "changes refuses with STORE_SNAPSHOT_EXPIRED", + "idempotency": false, + "operation": "changes", + "since": "1.0" + }, + "changes.raises.store-snapshot-not-found": { + "expectation": "changes refuses with STORE_SNAPSHOT_NOT_FOUND", + "idempotency": false, + "operation": "changes", + "since": "1.0" + }, + "changes.raises.store-statement-invalid": { + "expectation": "changes refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "changes", + "since": "1.0" + }, "changes.removed": { "expectation": "removed rows are counted; `since` after `snapshot` refused", "idempotency": false, @@ -76,26 +142,50 @@ "operation": "check_apart", "since": "1.0" }, - "clickhouse-catalog.forged-handle": { - "expectation": "a forged handle is refused", + "check-apart.raises.store-source-invalid": { + "expectation": "check_apart refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "check_apart", + "since": "1.0" + }, + "clickhouse-catalog.raises.store-parameter-invalid": { + "expectation": "clickhouse_catalog refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_catalog", "since": "1.0" }, - "clickhouse-columns.forged-handle": { - "expectation": "a forged handle is refused", + "clickhouse-columns.raises.source-not-found": { + "expectation": "clickhouse_columns refuses with SOURCE_NOT_FOUND", + "idempotency": false, + "operation": "clickhouse_columns", + "since": "1.0" + }, + "clickhouse-columns.raises.store-parameter-invalid": { + "expectation": "clickhouse_columns refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_columns", "since": "1.0" }, - "clickhouse-estimate.forged-handle": { - "expectation": "a forged handle is refused", + "clickhouse-columns.raises.unsupported-source-type": { + "expectation": "clickhouse_columns refuses with UNSUPPORTED_SOURCE_TYPE", + "idempotency": false, + "operation": "clickhouse_columns", + "since": "1.0" + }, + "clickhouse-estimate.raises.store-parameter-invalid": { + "expectation": "clickhouse_estimate refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_estimate", "since": "1.0" }, - "clickhouse-now.forged-handle": { - "expectation": "a forged handle is refused", + "clickhouse-estimate.raises.store-statement-invalid": { + "expectation": "clickhouse_estimate refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "clickhouse_estimate", + "since": "1.0" + }, + "clickhouse-now.raises.store-parameter-invalid": { + "expectation": "clickhouse_now refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_now", "since": "1.0" @@ -112,22 +202,34 @@ "operation": "close_existing_lake", "since": "1.0" }, + "close-existing-lake.raises.store-parameter-invalid": { + "expectation": "close_existing_lake refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "close_existing_lake", + "since": "1.0" + }, "close-lake.idempotent": { "expectation": "one key closes once", "idempotency": true, "operation": "close_lake", "since": "1.0" }, + "close-lake.raises.store-parameter-invalid": { + "expectation": "close_lake refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "close_lake", + "since": "1.0" + }, "close-read.idempotent": { "expectation": "one key lets a read go once", "idempotency": true, "operation": "close_read", "since": "1.0" }, - "commit.guarded": { - "expectation": "a view is refused; a forged transaction is refused", + "close-read.raises.store-parameter-invalid": { + "expectation": "close_read refuses with STORE_PARAMETER_INVALID", "idempotency": false, - "operation": "commit", + "operation": "close_read", "since": "1.0" }, "commit.idempotent": { @@ -136,8 +238,62 @@ "operation": "commit", "since": "1.0" }, - "connect-clickhouse.guarded": { - "expectation": "plain HTTP to another machine and a missing secret are refused", + "commit.raises.store-cancelled": { + "expectation": "commit refuses with STORE_CANCELLED", + "idempotency": false, + "operation": "commit", + "since": "1.0" + }, + "commit.raises.store-parameter-invalid": { + "expectation": "commit refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "commit", + "since": "1.0" + }, + "commit.raises.store-query-failed": { + "expectation": "commit refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "commit", + "since": "1.0" + }, + "commit.raises.store-timeout": { + "expectation": "commit refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "commit", + "since": "1.0" + }, + "commit.raises.store-transaction-conflict": { + "expectation": "commit refuses with STORE_TRANSACTION_CONFLICT", + "idempotency": false, + "operation": "commit", + "since": "1.0" + }, + "connect-clickhouse.raises.engine-setting-required": { + "expectation": "connect_clickhouse refuses with ENGINE_SETTING_REQUIRED", + "idempotency": false, + "operation": "connect_clickhouse", + "since": "1.0" + }, + "connect-clickhouse.raises.source-not-read-only": { + "expectation": "connect_clickhouse refuses with SOURCE_NOT_READ_ONLY", + "idempotency": false, + "operation": "connect_clickhouse", + "since": "1.0" + }, + "connect-clickhouse.raises.store-connection-failed": { + "expectation": "connect_clickhouse refuses with STORE_CONNECTION_FAILED", + "idempotency": false, + "operation": "connect_clickhouse", + "since": "1.0" + }, + "connect-clickhouse.raises.store-secret-missing": { + "expectation": "connect_clickhouse refuses with STORE_SECRET_MISSING", + "idempotency": false, + "operation": "connect_clickhouse", + "since": "1.0" + }, + "connect-clickhouse.raises.store-tls-required": { + "expectation": "connect_clickhouse refuses with STORE_TLS_REQUIRED", "idempotency": false, "operation": "connect_clickhouse", "since": "1.0" @@ -148,8 +304,8 @@ "operation": "current_snapshot", "since": "1.0" }, - "current-snapshot.forged-handle": { - "expectation": "a forged lake handle is refused", + "current-snapshot.raises.store-parameter-invalid": { + "expectation": "current_snapshot refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "current_snapshot", "since": "1.0" @@ -160,16 +316,52 @@ "operation": "describe_source", "since": "1.0" }, + "describe-source.raises.store-parameter-invalid": { + "expectation": "describe_source refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "describe_source", + "since": "1.0" + }, + "describe-source.raises.store-query-failed": { + "expectation": "describe_source refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "describe_source", + "since": "1.0" + }, + "describe-source.raises.store-source-invalid": { + "expectation": "describe_source refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "describe_source", + "since": "1.0" + }, + "describe-source.raises.store-statement-invalid": { + "expectation": "describe_source refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "describe_source", + "since": "1.0" + }, "describe.binder-only": { "expectation": "a SELECT's columns from the binder; anything else refused", "idempotency": false, "operation": "describe", "since": "1.0" }, - "execute.guarded": { - "expectation": "ATTACH, COPY TO and SET are refused in a transaction", + "describe.raises.store-parameter-invalid": { + "expectation": "describe refuses with STORE_PARAMETER_INVALID", "idempotency": false, - "operation": "execute", + "operation": "describe", + "since": "1.0" + }, + "describe.raises.store-query-failed": { + "expectation": "describe refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "describe", + "since": "1.0" + }, + "describe.raises.store-statement-invalid": { + "expectation": "describe refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "describe", "since": "1.0" }, "execute.idempotent": { @@ -178,14 +370,44 @@ "operation": "execute", "since": "1.0" }, - "existing-committed-at.forged-handle": { - "expectation": "a forged handle is refused", + "execute.raises.store-cancelled": { + "expectation": "execute refuses with STORE_CANCELLED", + "idempotency": false, + "operation": "execute", + "since": "1.0" + }, + "execute.raises.store-parameter-invalid": { + "expectation": "execute refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "execute", + "since": "1.0" + }, + "execute.raises.store-query-failed": { + "expectation": "execute refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "execute", + "since": "1.0" + }, + "execute.raises.store-statement-invalid": { + "expectation": "execute refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "execute", + "since": "1.0" + }, + "execute.raises.store-timeout": { + "expectation": "execute refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "execute", + "since": "1.0" + }, + "existing-committed-at.raises.store-parameter-invalid": { + "expectation": "existing_committed_at refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_committed_at", "since": "1.0" }, - "existing-latest.forged-handle": { - "expectation": "a forged handle is refused", + "existing-latest.raises.store-parameter-invalid": { + "expectation": "existing_latest refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_latest", "since": "1.0" @@ -196,20 +418,32 @@ "operation": "existing_latest", "since": "1.0" }, - "existing-table-columns.forged-handle": { - "expectation": "a forged handle is refused", + "existing-table-columns.raises.store-parameter-invalid": { + "expectation": "existing_table_columns refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_table_columns", "since": "1.0" }, - "existing-verify.forged-handle": { - "expectation": "a forged handle is refused", + "existing-table-columns.raises.store-source-invalid": { + "expectation": "existing_table_columns refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "existing_table_columns", + "since": "1.0" + }, + "existing-verify.raises.store-parameter-invalid": { + "expectation": "existing_verify refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "existing_verify", + "since": "1.0" + }, + "existing-verify.raises.store-source-invalid": { + "expectation": "existing_verify refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "existing_verify", "since": "1.0" }, - "existing-views.forged-handle": { - "expectation": "a forged handle is refused", + "existing-views.raises.store-parameter-invalid": { + "expectation": "existing_views refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_views", "since": "1.0" @@ -232,18 +466,60 @@ "operation": "json_frame", "since": "1.0" }, + "json-frame.raises.store-type-unsupported": { + "expectation": "json_frame refuses with STORE_TYPE_UNSUPPORTED", + "idempotency": false, + "operation": "json_frame", + "since": "1.0" + }, "json-rows.exact": { "expectation": "exact text, NULL as null, rows separated", "idempotency": false, "operation": "json_rows", "since": "1.0" }, + "json-rows.raises.store-type-unsupported": { + "expectation": "json_rows refuses with STORE_TYPE_UNSUPPORTED", + "idempotency": false, + "operation": "json_rows", + "since": "1.0" + }, "land.idempotent": { "expectation": "one key lands once", "idempotency": true, "operation": "land", "since": "1.0" }, + "land.raises.store-cancelled": { + "expectation": "land refuses with STORE_CANCELLED", + "idempotency": false, + "operation": "land", + "since": "1.0" + }, + "land.raises.store-parameter-invalid": { + "expectation": "land refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "land", + "since": "1.0" + }, + "land.raises.store-query-failed": { + "expectation": "land refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "land", + "since": "1.0" + }, + "land.raises.store-source-invalid": { + "expectation": "land refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "land", + "since": "1.0" + }, + "land.raises.store-timeout": { + "expectation": "land refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "land", + "since": "1.0" + }, "load-arrow.exact": { "expectation": "a DECIMAL and a NULL are loaded exactly", "idempotency": false, @@ -256,34 +532,70 @@ "operation": "load_arrow", "since": "1.0" }, + "load-arrow.raises.store-cancelled": { + "expectation": "load_arrow refuses with STORE_CANCELLED", + "idempotency": false, + "operation": "load_arrow", + "since": "1.0" + }, + "load-arrow.raises.store-parameter-invalid": { + "expectation": "load_arrow refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "load_arrow", + "since": "1.0" + }, + "load-arrow.raises.store-query-failed": { + "expectation": "load_arrow refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "load_arrow", + "since": "1.0" + }, + "load-arrow.raises.store-timeout": { + "expectation": "load_arrow refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "load_arrow", + "since": "1.0" + }, "location-components.parts": { "expectation": "object storage and local paths by parts; anything else refused", "idempotency": false, "operation": "location_components", "since": "1.0" }, + "location-components.raises.store-source-invalid": { + "expectation": "location_components refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "location_components", + "since": "1.0" + }, "locations-overlap.prefix": { "expectation": "inside overlaps, a sibling with the same prefix does not", "idempotency": false, "operation": "locations_overlap", "since": "1.0" }, + "locations-overlap.raises.store-source-invalid": { + "expectation": "locations_overlap refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "locations_overlap", + "since": "1.0" + }, "maintain.idempotent": { "expectation": "one key maintains once and answers the same", "idempotency": true, "operation": "maintain", "since": "1.0" }, - "maintain.naive-cutoff": { - "expectation": "a cutoff without a zone is refused", + "maintain.raises.store-busy": { + "expectation": "maintain refuses with STORE_BUSY", "idempotency": false, "operation": "maintain", "since": "1.0" }, - "open-existing-lake.guarded": { - "expectation": "encryption and a missing protected set are refused", + "maintain.raises.store-parameter-invalid": { + "expectation": "maintain refuses with STORE_PARAMETER_INVALID", "idempotency": false, - "operation": "open_existing_lake", + "operation": "maintain", "since": "1.0" }, "open-existing-lake.idempotent": { @@ -292,20 +604,98 @@ "operation": "open_existing_lake", "since": "1.0" }, + "open-existing-lake.raises.engine-version-untested": { + "expectation": "open_existing_lake refuses with ENGINE_VERSION_UNTESTED", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.raises.store-connection-failed": { + "expectation": "open_existing_lake refuses with STORE_CONNECTION_FAILED", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.raises.store-limits-required": { + "expectation": "open_existing_lake refuses with STORE_LIMITS_REQUIRED", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.raises.store-location-invalid": { + "expectation": "open_existing_lake refuses with STORE_LOCATION_INVALID", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.raises.store-secret-missing": { + "expectation": "open_existing_lake refuses with STORE_SECRET_MISSING", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.raises.store-source-invalid": { + "expectation": "open_existing_lake refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, + "open-existing-lake.raises.store-tls-required": { + "expectation": "open_existing_lake refuses with STORE_TLS_REQUIRED", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, "open-lake.idempotent": { "expectation": "one key opens one lake", "idempotency": true, "operation": "open_lake", "since": "1.0" }, - "open-lake.limits-required": { - "expectation": "a lake without every limit is refused (law 7)", + "open-lake.raises.source-not-found": { + "expectation": "open_lake refuses with SOURCE_NOT_FOUND", "idempotency": false, "operation": "open_lake", "since": "1.0" }, - "open-lake.source-not-found": { - "expectation": "a missing source file is refused before anything opens", + "open-lake.raises.store-catalog-unavailable": { + "expectation": "open_lake refuses with STORE_CATALOG_UNAVAILABLE", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.raises.store-limits-required": { + "expectation": "open_lake refuses with STORE_LIMITS_REQUIRED", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.raises.store-location-invalid": { + "expectation": "open_lake refuses with STORE_LOCATION_INVALID", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.raises.store-secret-inline": { + "expectation": "open_lake refuses with STORE_SECRET_INLINE", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.raises.store-secret-missing": { + "expectation": "open_lake refuses with STORE_SECRET_MISSING", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.raises.store-source-invalid": { + "expectation": "open_lake refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, + "open-lake.raises.store-tls-required": { + "expectation": "open_lake refuses with STORE_TLS_REQUIRED", "idempotency": false, "operation": "open_lake", "since": "1.0" @@ -316,6 +706,30 @@ "operation": "read_batch", "since": "1.0" }, + "read-batch.raises.store-cancelled": { + "expectation": "read_batch refuses with STORE_CANCELLED", + "idempotency": false, + "operation": "read_batch", + "since": "1.0" + }, + "read-batch.raises.store-parameter-invalid": { + "expectation": "read_batch refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "read_batch", + "since": "1.0" + }, + "read-batch.raises.store-query-failed": { + "expectation": "read_batch refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "read_batch", + "since": "1.0" + }, + "read-batch.raises.store-timeout": { + "expectation": "read_batch refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "read_batch", + "since": "1.0" + }, "rollback.discards": { "expectation": "a rolled-back transaction writes nothing and is let go", "idempotency": false, @@ -328,20 +742,62 @@ "operation": "rollback", "since": "1.0" }, + "rollback.raises.store-parameter-invalid": { + "expectation": "rollback refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "rollback", + "since": "1.0" + }, "snapshot.kept-or-none": { "expectation": "a kept snapshot is found; an unknown or negative id is None", "idempotency": false, "operation": "snapshot", "since": "1.0" }, - "start-clickhouse-read.forged-handle": { - "expectation": "a forged handle is refused", + "snapshot.raises.store-parameter-invalid": { + "expectation": "snapshot refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "snapshot", + "since": "1.0" + }, + "start-clickhouse-read.raises.store-parameter-invalid": { + "expectation": "start_clickhouse_read refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "start_clickhouse_read", + "since": "1.0" + }, + "start-clickhouse-read.raises.store-query-failed": { + "expectation": "start_clickhouse_read refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "start_clickhouse_read", "since": "1.0" }, - "start-existing-read.forged-handle": { - "expectation": "a forged handle is refused", + "start-clickhouse-read.raises.store-statement-invalid": { + "expectation": "start_clickhouse_read refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "start_clickhouse_read", + "since": "1.0" + }, + "start-existing-read.raises.store-parameter-invalid": { + "expectation": "start_existing_read refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "start_existing_read", + "since": "1.0" + }, + "start-existing-read.raises.store-query-failed": { + "expectation": "start_existing_read refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "start_existing_read", + "since": "1.0" + }, + "start-existing-read.raises.store-source-invalid": { + "expectation": "start_existing_read refuses with STORE_SOURCE_INVALID", + "idempotency": false, + "operation": "start_existing_read", + "since": "1.0" + }, + "start-existing-read.raises.store-statement-invalid": { + "expectation": "start_existing_read refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "start_existing_read", "since": "1.0" @@ -352,14 +808,38 @@ "operation": "start_read", "since": "1.0" }, - "start-read.guarded": { - "expectation": "file readers, writes, two statements and unknown snapshots are refused", + "start-read.parameters": { + "expectation": "a parameter is bound by name with its type", "idempotency": false, "operation": "start_read", "since": "1.0" }, - "start-read.parameters": { - "expectation": "a parameter is bound by name with its type", + "start-read.raises.store-busy": { + "expectation": "start_read refuses with STORE_BUSY", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "start-read.raises.store-parameter-invalid": { + "expectation": "start_read refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "start-read.raises.store-snapshot-expired": { + "expectation": "start_read refuses with STORE_SNAPSHOT_EXPIRED", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "start-read.raises.store-snapshot-not-found": { + "expectation": "start_read refuses with STORE_SNAPSHOT_NOT_FOUND", + "idempotency": false, + "operation": "start_read", + "since": "1.0" + }, + "start-read.raises.store-statement-invalid": { + "expectation": "start_read refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "start_read", "since": "1.0" @@ -370,26 +850,116 @@ "operation": "statistics", "since": "1.0" }, + "statistics.raises.store-busy": { + "expectation": "statistics refuses with STORE_BUSY", + "idempotency": false, + "operation": "statistics", + "since": "1.0" + }, + "statistics.raises.store-parameter-invalid": { + "expectation": "statistics refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "statistics", + "since": "1.0" + }, + "statistics.raises.store-snapshot-expired": { + "expectation": "statistics refuses with STORE_SNAPSHOT_EXPIRED", + "idempotency": false, + "operation": "statistics", + "since": "1.0" + }, + "statistics.raises.store-snapshot-not-found": { + "expectation": "statistics refuses with STORE_SNAPSHOT_NOT_FOUND", + "idempotency": false, + "operation": "statistics", + "since": "1.0" + }, + "statistics.raises.store-statement-invalid": { + "expectation": "statistics refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "statistics", + "since": "1.0" + }, "table-columns.as-of": { "expectation": "the columns as of the snapshot asked", "idempotency": false, "operation": "table_columns", "since": "1.0" }, + "table-columns.raises.store-busy": { + "expectation": "table_columns refuses with STORE_BUSY", + "idempotency": false, + "operation": "table_columns", + "since": "1.0" + }, + "table-columns.raises.store-parameter-invalid": { + "expectation": "table_columns refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "table_columns", + "since": "1.0" + }, + "table-columns.raises.store-snapshot-expired": { + "expectation": "table_columns refuses with STORE_SNAPSHOT_EXPIRED", + "idempotency": false, + "operation": "table_columns", + "since": "1.0" + }, + "table-columns.raises.store-snapshot-not-found": { + "expectation": "table_columns refuses with STORE_SNAPSHOT_NOT_FOUND", + "idempotency": false, + "operation": "table_columns", + "since": "1.0" + }, "tables.lower-cased": { "expectation": "every table, lower-cased, in order", "idempotency": false, "operation": "tables", "since": "1.0" }, + "tables.raises.store-parameter-invalid": { + "expectation": "tables refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "tables", + "since": "1.0" + }, "value.first-cell": { "expectation": "the first cell as text; None when there is no row", "idempotency": false, "operation": "value", "since": "1.0" + }, + "value.raises.store-cancelled": { + "expectation": "value refuses with STORE_CANCELLED", + "idempotency": false, + "operation": "value", + "since": "1.0" + }, + "value.raises.store-parameter-invalid": { + "expectation": "value refuses with STORE_PARAMETER_INVALID", + "idempotency": false, + "operation": "value", + "since": "1.0" + }, + "value.raises.store-query-failed": { + "expectation": "value refuses with STORE_QUERY_FAILED", + "idempotency": false, + "operation": "value", + "since": "1.0" + }, + "value.raises.store-statement-invalid": { + "expectation": "value refuses with STORE_STATEMENT_INVALID", + "idempotency": false, + "operation": "value", + "since": "1.0" + }, + "value.raises.store-timeout": { + "expectation": "value refuses with STORE_TIMEOUT", + "idempotency": false, + "operation": "value", + "since": "1.0" } }, - "recorded": 58 + "recorded": 153 }, "name": "store", "operations": { @@ -397,41 +967,20 @@ "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TIMEOUT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" @@ -444,41 +993,17 @@ "capability": "read", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "read", "type": "orvex.contracts.store.values.ReadHandle" @@ -491,41 +1016,17 @@ "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" @@ -538,47 +1039,27 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", "STORE_SNAPSHOT_EXPIRED", "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.ChangesRequest" @@ -590,66 +1071,42 @@ "check_apart": { "capability": "separation", "changes_state": false, - "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "errors": [ + "STORE_SOURCE_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "name", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "data_path", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "catalog", "type": "str | None" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "protected", "type": "tuple[str, ...]" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "protected_files", "type": "tuple[str, ...]" @@ -662,41 +1119,17 @@ "capability": "clickhouse", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "engine", "type": "orvex.contracts.store.values.ClickHouseHandle" @@ -709,47 +1142,25 @@ "capability": "clickhouse", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", "UNSUPPORTED_SOURCE_TYPE" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "engine", "type": "orvex.contracts.store.values.ClickHouseHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "table", "type": "str" @@ -762,53 +1173,30 @@ "capability": "clickhouse", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "engine", "type": "orvex.contracts.store.values.ClickHouseHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "sql", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "parameters", "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" @@ -821,41 +1209,17 @@ "capability": "clickhouse", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "engine", "type": "orvex.contracts.store.values.ClickHouseHandle" @@ -867,42 +1231,16 @@ "close_clickhouse": { "capability": "clickhouse", "changes_state": false, - "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" - ], + "errors": [], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "engine", "type": "orvex.contracts.store.values.ClickHouseHandle" @@ -915,41 +1253,17 @@ "capability": "existing-lake", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" @@ -962,41 +1276,17 @@ "capability": "lake", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" @@ -1009,41 +1299,17 @@ "capability": "read", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "read", "type": "orvex.contracts.store.values.ReadHandle" @@ -1056,41 +1322,21 @@ "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TRANSACTION_CONFLICT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" @@ -1104,46 +1350,26 @@ "changes_state": false, "errors": [ "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TLS_REQUIRED" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "target", "type": "orvex.contracts.store.values.ClickHouseTarget" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "budget", "type": "orvex.contracts.store.values.Budget" @@ -1156,41 +1382,17 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" @@ -1203,53 +1405,31 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "sql", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "parameters", "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" @@ -1262,59 +1442,38 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "name", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "sql", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "parameters", "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" @@ -1327,47 +1486,27 @@ "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", + "STORE_CANCELLED", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TIMEOUT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.StatementRequest" @@ -1380,47 +1519,23 @@ "capability": "existing-lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "snapshot", "type": "int" @@ -1433,41 +1548,17 @@ "capability": "existing-lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" @@ -1480,47 +1571,24 @@ "capability": "existing-lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_SOURCE_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "snapshot", "type": "orvex.contracts.store.v1.Snapshot" @@ -1533,53 +1601,30 @@ "capability": "existing-lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_SOURCE_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "snapshot", "type": "orvex.contracts.store.v1.Snapshot" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "current", "type": "orvex.contracts.store.v1.ExistingLakeAttachment" @@ -1592,47 +1637,23 @@ "capability": "existing-lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "snapshot", "type": "orvex.contracts.store.v1.Snapshot" @@ -1644,36 +1665,10 @@ "held": { "capability": "lake", "changes_state": false, - "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" - ], + "errors": [], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" @@ -1685,42 +1680,16 @@ "is_object_storage": { "capability": "separation", "changes_state": false, - "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" - ], + "errors": [], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "location", "type": "str" @@ -1733,44 +1702,20 @@ "capability": "encoding", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TYPE_UNSUPPORTED" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "schema", - "type": "bytes" + "type": "pyarrow.lib.Table" } ], "returns": "orvex.contracts.store.values.JsonFrame", @@ -1780,47 +1725,23 @@ "capability": "encoding", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TYPE_UNSUPPORTED" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", - "name": "batch", - "type": "bytes" + "name": "rows", + "type": "pyarrow.lib.Table" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "first", "type": "bool" @@ -1833,77 +1754,57 @@ "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TIMEOUT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "source", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "sql", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "parameters", "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "schema", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "table", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "timeout_ms", "type": "int | None" @@ -1916,124 +1817,79 @@ "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TIMEOUT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "schema", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "table", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "data", - "type": "bytes" + "type": "pyarrow.lib.Table" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "columns", "type": "tuple[tuple[str, str], ...]" }, - { - "default": false, - "kind": "positional-or-keyword", - "name": "timeout_ms", - "type": "int | None" - } - ], - "returns": "int", - "since": "1.0" - }, - "location_components": { - "capability": "separation", - "changes_state": false, - "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + { + "default": null, + "kind": "positional-or-keyword", + "name": "timeout_ms", + "type": "int | None" + } + ], + "returns": "int", + "since": "1.0" + }, + "location_components": { + "capability": "separation", + "changes_state": false, + "errors": [ + "STORE_SOURCE_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "name", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "location", "type": "str" @@ -2046,53 +1902,29 @@ "capability": "separation", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_SOURCE_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "name", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "first", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "second", "type": "str" @@ -2105,53 +1937,30 @@ "capability": "maintenance", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "expire_before", "type": "datetime.datetime" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "orphans_before", "type": "datetime.datetime" @@ -2164,47 +1973,29 @@ "capability": "existing-lake", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", "STORE_LIMITS_REQUIRED", "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TLS_REQUIRED" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "attachment", "type": "orvex.contracts.store.v1.ExistingLakeAttachment" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "limits", "type": "orvex.contracts.store.values.ExistingLakeLimits" @@ -2217,53 +2008,36 @@ "capability": "lake", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", "STORE_LIMITS_REQUIRED", "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", "STORE_SECRET_INLINE", "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TLS_REQUIRED" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "location", "type": "orvex.contracts.store.v1.LocalLakeLocation | orvex.contracts.store.v1.SqliteLakeLocation | orvex.contracts.store.v1.PostgresLakeLocation" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "limits", "type": "orvex.contracts.store.values.Limits" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "sources", "type": "tuple[orvex.contracts.store.v1.FileSourceAttachment | orvex.contracts.store.v1.SqliteSourceAttachment | orvex.contracts.store.v1.PostgresSourceAttachment, ...]" @@ -2276,88 +2050,43 @@ "capability": "read", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TIMEOUT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "read", "type": "orvex.contracts.store.values.ReadHandle" } ], - "returns": "bytes | None", + "returns": "pyarrow.lib.Table | None", "since": "1.0" }, "rollback": { "capability": "write", "changes_state": true, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" @@ -2370,47 +2099,23 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "snapshot_id", "type": "int" @@ -2423,65 +2128,43 @@ "capability": "clickhouse", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "engine", "type": "orvex.contracts.store.values.ClickHouseHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "sql", "type": "str" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "parameters", "type": "tuple[orvex.contracts.store.v1.Parameter, ...]" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "columns", "type": "tuple[tuple[str, str], ...]" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "timeout_ms", "type": "int | None" @@ -2494,53 +2177,32 @@ "capability": "existing-lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.ExistingLakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.ReadRequest" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "committed_at", "type": "datetime.datetime" @@ -2553,47 +2215,27 @@ "capability": "read", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", "STORE_SNAPSHOT_EXPIRED", "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.ReadRequest" @@ -2606,47 +2248,27 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", "STORE_SNAPSHOT_EXPIRED", "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_STATEMENT_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.StatisticsRequest" @@ -2659,47 +2281,26 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_SNAPSHOT_NOT_FOUND" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.TablesRequest" @@ -2712,41 +2313,17 @@ "capability": "lake", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", - "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", - "STORE_PARAMETER_INVALID", - "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", - "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_PARAMETER_INVALID" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "lake", "type": "orvex.contracts.store.values.LakeHandle" @@ -2759,47 +2336,27 @@ "capability": "write", "changes_state": false, "errors": [ - "ENGINE_SETTING_REQUIRED", - "ENGINE_VERSION_UNTESTED", - "SOURCE_NOT_FOUND", - "SOURCE_NOT_READ_ONLY", - "STORE_BUSY", "STORE_CANCELLED", - "STORE_CATALOG_UNAVAILABLE", - "STORE_CONNECTION_FAILED", - "STORE_EXTENSION_UNAVAILABLE", - "STORE_LIMITS_REQUIRED", - "STORE_LOCATION_INVALID", "STORE_PARAMETER_INVALID", "STORE_QUERY_FAILED", - "STORE_SECRET_INLINE", - "STORE_SECRET_MISSING", - "STORE_SNAPSHOT_EXPIRED", - "STORE_SNAPSHOT_NOT_FOUND", - "STORE_SOURCE_FAILED", - "STORE_SOURCE_INVALID", "STORE_STATEMENT_INVALID", - "STORE_TIMEOUT", - "STORE_TLS_REQUIRED", - "STORE_TRANSACTION_CONFLICT", - "STORE_TYPE_UNSUPPORTED", - "UNSUPPORTED_SOURCE_TYPE" + "STORE_TIMEOUT" ], "parameters": [ { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "envelope", "type": "orvex.contracts.envelope.CallEnvelope" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "transaction", "type": "orvex.contracts.store.values.TransactionHandle" }, { - "default": false, + "default": null, "kind": "positional-or-keyword", "name": "request", "type": "orvex.contracts.store.v1.StatementRequest" diff --git a/packages/contracts/src/orvex/contracts/store/interface.py b/packages/contracts/src/orvex/contracts/store/interface.py index f1f31326..dfe82065 100644 --- a/packages/contracts/src/orvex/contracts/store/interface.py +++ b/packages/contracts/src/orvex/contracts/store/interface.py @@ -294,8 +294,8 @@ def check_apart( } -# Every code of `contracts/store/v1`: each operation reaches DuckDB, a source or a check that -# may refuse with any of them, so each declares them all. +# Every code of `contracts/store/v1` is the contract's; each operation declares only the codes +# it raises, and the kit proves every one (`orvex.contracts.store.kit_codes`). CODES = ( "STORE_SNAPSHOT_NOT_FOUND", "STORE_SNAPSHOT_EXPIRED", @@ -323,10 +323,96 @@ def check_apart( "ENGINE_VERSION_UNTESTED", "ENGINE_SETTING_REQUIRED", ) +HANDLE = "STORE_PARAMETER_INVALID" # a handle that names nothing held, or a parameter refused +SNAPSHOT = ("STORE_SNAPSHOT_NOT_FOUND", "STORE_SNAPSHOT_EXPIRED") +SLOT = ("STORE_BUSY",) +STATEMENT = ("STORE_STATEMENT_INVALID", "STORE_QUERY_FAILED") +ERRORS_OF: dict[str, tuple[str, ...]] = { + "open_lake": ( + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_SECRET_MISSING", + "STORE_SECRET_INLINE", + "STORE_TLS_REQUIRED", + "STORE_CATALOG_UNAVAILABLE", + "SOURCE_NOT_FOUND", + "STORE_SOURCE_INVALID", + ), + "close_lake": (HANDLE,), + "held": (), + "current_snapshot": (HANDLE,), + "snapshot": (HANDLE,), + "tables": (HANDLE,), + "describe": (HANDLE, *STATEMENT), + "describe_source": (HANDLE, *STATEMENT, "STORE_SOURCE_INVALID"), + "statistics": (HANDLE, *SNAPSHOT, *SLOT, "STORE_STATEMENT_INVALID"), + "changes": (HANDLE, *SNAPSHOT, *SLOT, "STORE_STATEMENT_INVALID"), + "table_columns": (HANDLE, *SNAPSHOT, *SLOT), + "start_read": (HANDLE, *SNAPSHOT, *SLOT, "STORE_STATEMENT_INVALID"), + "read_batch": (HANDLE, "STORE_QUERY_FAILED", "STORE_TIMEOUT", "STORE_CANCELLED"), + "cancel_read": (HANDLE,), + "close_read": (HANDLE,), + "begin": (HANDLE, *SLOT, "STORE_STATEMENT_INVALID", "STORE_TIMEOUT"), + "execute": (HANDLE, *STATEMENT, "STORE_CANCELLED", "STORE_TIMEOUT"), + "value": (HANDLE, *STATEMENT, "STORE_CANCELLED", "STORE_TIMEOUT"), + "land": ( + HANDLE, + "STORE_SOURCE_INVALID", + "STORE_QUERY_FAILED", + "STORE_CANCELLED", + "STORE_TIMEOUT", + ), + "load_arrow": (HANDLE, "STORE_QUERY_FAILED", "STORE_CANCELLED", "STORE_TIMEOUT"), + "cancel_transaction": (HANDLE,), + "commit": ( + HANDLE, + "STORE_CANCELLED", + "STORE_QUERY_FAILED", + "STORE_TIMEOUT", + "STORE_TRANSACTION_CONFLICT", + ), + "rollback": (HANDLE,), + "maintain": (HANDLE, *SLOT), + "open_existing_lake": ( + "STORE_LIMITS_REQUIRED", + "STORE_LOCATION_INVALID", + "STORE_SECRET_MISSING", + "STORE_TLS_REQUIRED", + "STORE_CONNECTION_FAILED", + "STORE_SOURCE_INVALID", + "ENGINE_VERSION_UNTESTED", + ), + "close_existing_lake": (HANDLE,), + "existing_latest": (HANDLE,), + "existing_verify": (HANDLE, "STORE_SOURCE_INVALID"), + "existing_committed_at": (HANDLE,), + "existing_table_columns": (HANDLE, "STORE_SOURCE_INVALID"), + "existing_views": (HANDLE,), + "start_existing_read": (HANDLE, *STATEMENT, "STORE_SOURCE_INVALID"), + "connect_clickhouse": ( + "STORE_TLS_REQUIRED", + "STORE_SECRET_MISSING", + "STORE_CONNECTION_FAILED", + "SOURCE_NOT_READ_ONLY", + "ENGINE_SETTING_REQUIRED", + ), + "close_clickhouse": (), + "clickhouse_columns": (HANDLE, "SOURCE_NOT_FOUND", "UNSUPPORTED_SOURCE_TYPE"), + "clickhouse_catalog": (HANDLE,), + "clickhouse_estimate": (HANDLE, "STORE_STATEMENT_INVALID"), + "clickhouse_now": (HANDLE,), + "start_clickhouse_read": (HANDLE, *STATEMENT), + "json_frame": ("STORE_TYPE_UNSUPPORTED",), + "json_rows": ("STORE_TYPE_UNSUPPORTED",), + "is_object_storage": (), + "location_components": ("STORE_SOURCE_INVALID",), + "locations_overlap": ("STORE_SOURCE_INVALID",), + "check_apart": ("STORE_SOURCE_INVALID",), +} def _operation(name: str, capability: str, *, changes_state: bool = False) -> Operation: - return Operation(name, capability, CODES, changes_state=changes_state) + return Operation(name, capability, ERRORS_OF[name], changes_state=changes_state) STORE = InterfaceContract( diff --git a/packages/contracts/src/orvex/contracts/store/kit.py b/packages/contracts/src/orvex/contracts/store/kit.py index ca306b57..72dd64f5 100644 --- a/packages/contracts/src/orvex/contracts/store/kit.py +++ b/packages/contracts/src/orvex/contracts/store/kit.py @@ -1,10 +1,11 @@ """The store's conformance kit (#486 G1): every operation of `orvex.contracts.store.interface`, its exactness, its fail-closed refusals and the security cases consumers rely on (the SQL guard, the sandbox, forged handles, TLS and secrets), and one idempotency case per operation that -changes state. No skips; its case count is recorded. +changes state, and one case per error code each operation declares (`kit_codes`). No skips; its +case count is recorded. -Every case runs on local folders and SQLite files only, so the kit needs no server. ClickHouse -is reached in its refusals and its handles; its answers are the store's own tests' (CI). +Most cases run on local folders and SQLite files; the ClickHouse cases take a server from the +environment (`kit_services`) and fail, never skip, without one. """ from __future__ import annotations @@ -19,6 +20,7 @@ from orvex.contracts.kit import Idempotent, Kit, KitCase from orvex.contracts.kit_client import KitClient from orvex.contracts.store.interface import STORE +from orvex.contracts.store.kit_codes import code_cases from orvex.contracts.store.kit_support import ( EXISTING_LIMITS, LIMITS, @@ -45,10 +47,8 @@ from orvex.contracts.store.values import ( Budget, ClickHouseHandle, - ClickHouseTarget, ExistingLakeHandle, LakeHandle, - Limits, ReadHandle, TransactionHandle, ) @@ -86,17 +86,6 @@ def _lakes(client: KitClient) -> int: return len(client.held().lakes) -def _open_refuses_missing_limits(client: KitClient) -> None: - refused( - "STORE_LIMITS_REQUIRED", lambda: client.open_lake(local_location(), Limits(None, None), ()) - ) - - -def _open_refuses_missing_source(client: KitClient) -> None: - source = sqlite_source().model_copy(update={"path": "/nonexistent/kit.sqlite"}) - refused("SOURCE_NOT_FOUND", lambda: client.open_lake(local_location(), LIMITS, (source,))) - - _closing = Scenario(open_lake) @@ -112,10 +101,6 @@ def _held_lists(client: KitClient) -> None: refused(PARAMETER, lambda: client.current_snapshot(lake)) -def _forged_lake(client: KitClient) -> None: - refused(PARAMETER, lambda: client.current_snapshot(FORGED_LAKE)) - - def _current_after_commit(client: KitClient) -> None: lake = open_lake(client) committed = write(client, lake, "CREATE TABLE t AS SELECT 1 AS a") @@ -141,14 +126,12 @@ def _describe(client: KitClient) -> None: lake = open_lake(client) found = client.describe(lake, "SELECT 1.50::DECIMAL(18,2) AS a, NULL::VARCHAR AS b", ()) check(found == (("a", "DECIMAL(18,2)"), ("b", "VARCHAR")), f"described: {found}") - refused(STATEMENT, lambda: client.describe(lake, "DELETE FROM x", ())) def _describe_source(client: KitClient) -> None: lake = open_lake(client, (sqlite_source(),)) found = client.describe_source(lake, "field", "SELECT * FROM meters", ()) check(found == (("id", "BIGINT"), ("name", "VARCHAR")), f"source columns: {found}") - refused(SOURCE, lambda: client.describe_source(lake, "nope", "SELECT 1", ())) def _statistics(client: KitClient) -> None: @@ -160,10 +143,6 @@ def _statistics(client: KitClient) -> None: (table,) = client.statistics(lake, request).tables counted = table.inlined_rows + sum(file.row_count for file in table.files) check(counted == 3, f"three rows are counted: {table}") - unknown = StatisticsRequest.model_validate( - {"kind": "statistics", "tables": ["main.nope"], "snapshot": snapshot.id} - ) - refused(STATEMENT, lambda: client.statistics(lake, unknown)) def _changes(client: KitClient) -> None: @@ -175,10 +154,6 @@ def _changes(client: KitClient) -> None: ) (table,) = client.changes(lake, request).tables check(table.changed and table.removed_rows == 1, f"one row removed: {table}") - backwards = ChangesRequest.model_validate( - {"kind": "changes", "tables": ["main.t"], "since": after.id, "snapshot": since.id} - ) - refused(STATEMENT, lambda: client.changes(lake, backwards)) def _table_columns(client: KitClient) -> None: @@ -203,18 +178,6 @@ def _exact(client: KitClient) -> None: check(row["missing"] is None, f"NULL is never zero: {row}") -def _read_refusals(client: KitClient) -> None: - lake = open_lake(client) - snapshot = commit(client, lake, "CREATE TABLE t AS SELECT 1 AS a").id - for sql in ("SELECT * FROM read_csv('/etc/passwd')", "DELETE FROM t", "SELECT 1; SELECT 2"): - request = read_request(sql, snapshot) - refused(STATEMENT, functools.partial(client.start_read, lake, request)) - refused( - "STORE_SNAPSHOT_NOT_FOUND", - lambda: client.start_read(lake, read_request("SELECT 1", 999_999)), - ) - - def _read_parameters(client: KitClient) -> None: lake = open_lake(client) snapshot = commit(client, lake, "CREATE TABLE t AS SELECT * FROM range(5) AS r(a)").id @@ -240,7 +203,6 @@ def _read_ends(client: KitClient) -> None: check(seen == [0, 1, 2], f"every row, once: {seen}") check(client.read_batch(read) is None, "an ended read stays ended") client.close_read(read) - refused(PARAMETER, lambda: client.read_batch(FORGED_READ)) _cancelling = Scenario( @@ -300,19 +262,6 @@ def _count(client: KitClient, transaction: TransactionHandle, table: str) -> str return found -def _write_refusals(client: KitClient) -> None: - lake = open_lake(client) - for sql in ( - "ATTACH '/tmp/kit.db' AS other", - "COPY (SELECT 1) TO '/tmp/kit.csv'", - "SET threads = 8", - ): - transaction = client.begin(lake) - refused(STATEMENT, functools.partial(client.execute, transaction, statement(sql))) - client.rollback(transaction) - check(client.tables(lake) == (), "nothing was written") - - def _value(client: KitClient) -> None: lake = open_lake(client) write(client, lake, "CREATE TABLE t AS SELECT 1.50::DECIMAL(18,2) AS a") @@ -349,14 +298,6 @@ def _cancel_rolls_back(client: KitClient) -> None: check(client.tables(lake) == (), "a cancelled transaction writes nothing") -def _commit_refuses_views(client: KitClient) -> None: - lake = open_lake(client) - transaction = client.begin(lake) - refused(STATEMENT, lambda: client.execute(transaction, statement("CREATE VIEW v AS SELECT 1"))) - client.rollback(transaction) - refused(PARAMETER, lambda: client.commit(FORGED_TRANSACTION)) - - def _uncommitted(client: KitClient) -> tuple[LakeHandle, TransactionHandle]: lake = open_lake(client) transaction = client.begin(lake) @@ -403,12 +344,6 @@ def _expired(client: KitClient) -> bool: return client.snapshot(lake, first) is None -def _maintain_refuses_naive(client: KitClient) -> None: - lake = open_lake(client) - naive = dt.datetime(2020, 1, 1) - refused(PARAMETER, lambda: client.maintain(lake, naive, naive)) - - # --- existing lake ------------------------------------------------------------------------------- _attaching = Scenario(lambda client: existing_lake(client)[1]) @@ -438,70 +373,15 @@ def _existing_reads(client: KitClient) -> None: client.close_existing_lake(lake) -def _existing_refusals(client: KitClient) -> None: - _, attachment = existing_lake(client) - encrypted = attachment.model_copy( - update={"location": attachment.location.model_copy(update={"encrypted": True})} - ) - refused("STORE_LOCATION_INVALID", lambda: client.open_existing_lake(encrypted, EXISTING_LIMITS)) - unprotected = attachment.model_copy(update={"protected_locations": None}) - refused(SOURCE, lambda: client.open_existing_lake(unprotected, EXISTING_LIMITS)) - refused(PARAMETER, lambda: client.existing_latest(FORGED_EXISTING)) - - -def _existing_forged(operation: str) -> KitCase: - def run(client: KitClient) -> None: - lake, attachment = existing_lake(client) - latest = client.existing_latest(lake) - arguments: dict[str, tuple[object, ...]] = { - "existing_latest": (), - "existing_verify": (latest, attachment), - "existing_committed_at": (latest.id,), - "existing_table_columns": (latest,), - "existing_views": (latest,), - "start_existing_read": (read_request("SELECT 1", latest.id), latest.committed_at), - } - envelope = client.envelope(operation) - refused( - PARAMETER, - lambda: client.send(operation, envelope, FORGED_EXISTING, *arguments[operation]), - ) - - return KitCase( - f"{operation.replace('_', '-')}.forged-handle", operation, "a forged handle is refused", run - ) - - # --- clickhouse ---------------------------------------------------------------------------------- -def _clickhouse_refusals(client: KitClient) -> None: - remote = ClickHouseTarget( - "clickhouse.example", 8123, "db", "reader", "KIT_UNSET_PASSWORD", tls=False - ) - refused("STORE_TLS_REQUIRED", lambda: client.connect_clickhouse(remote, BUDGET)) - local = ClickHouseTarget("localhost", 8123, "db", "reader", "KIT_UNSET_PASSWORD_486", tls=False) - refused("STORE_SECRET_MISSING", lambda: client.connect_clickhouse(local, BUDGET)) - - def _clickhouse_close(client: KitClient) -> None: client.close_clickhouse(FORGED_ENGINE) client.close_clickhouse(FORGED_ENGINE) check(client.held().clickhouse == (), "closing what is not held changes nothing") -def _clickhouse_forged(operation: str, *arguments: object) -> KitCase: - def run(client: KitClient) -> None: - refused( - PARAMETER, - lambda: client.send(operation, client.envelope(operation), FORGED_ENGINE, *arguments), - ) - - return KitCase( - f"{operation.replace('_', '-')}.forged-handle", operation, "a forged handle is refused", run - ) - - # --- encoding ------------------------------------------------------------------------------------ @@ -525,8 +405,6 @@ def _json_frame(client: KitClient) -> None: '{"name": "h", "type": "HUGEINT"}],"rows":[' ) check(found == expected, f"the head: {found}") - bare = pa.schema([("x", pa.int64())]).empty_table() - refused("STORE_TYPE_UNSUPPORTED", lambda: client.json_frame(bare)) def _json_rows(client: KitClient) -> None: @@ -560,8 +438,6 @@ def _overlap(client: KitClient) -> None: def _apart(client: KitClient) -> None: - root = str(local_location().path) - refused(SOURCE, lambda: client.check_apart("c", root + "/data", None, (root,), ())) client.check_apart("c", "s3://bucket/lake/", None, ("s3://bucket/other/",), ()) @@ -579,18 +455,6 @@ def _case( "one key opens one lake", Idempotent(_open_call, _lakes), ), - _case( - "open-lake.limits-required", - "open_lake", - "a lake without every limit is refused (law 7)", - _open_refuses_missing_limits, - ), - _case( - "open-lake.source-not-found", - "open_lake", - "a missing source file is refused before anything opens", - _open_refuses_missing_source, - ), KitCase( "close-lake.idempotent", "close_lake", @@ -603,12 +467,6 @@ def _case( "what is held is listed; a closed lake is let go and refused", _held_lists, ), - _case( - "current-snapshot.forged-handle", - "current_snapshot", - "a forged lake handle is refused", - _forged_lake, - ), _case( "current-snapshot.after-commit", "current_snapshot", @@ -659,12 +517,6 @@ def _case( "DECIMAL, HUGEINT, TIMESTAMPTZ and NULL are read exactly", _exact, ), - _case( - "start-read.guarded", - "start_read", - "file readers, writes, two statements and unknown snapshots are refused", - _read_refusals, - ), _case( "start-read.parameters", "start_read", @@ -713,12 +565,6 @@ def _case( lambda c: _count(c, _executing.of(c), "t"), ), ), - _case( - "execute.guarded", - "execute", - "ATTACH, COPY TO and SET are refused in a transaction", - _write_refusals, - ), _case("value.first-cell", "value", "the first cell as text; None when there is no row", _value), KitCase( "land.idempotent", @@ -779,12 +625,6 @@ def _case( "one key commits once and answers the same", Idempotent(lambda c, e: c.send("commit", e, _committing.of(c)[1]), _committed), ), - _case( - "commit.guarded", - "commit", - "a view is refused; a forged transaction is refused", - _commit_refuses_views, - ), KitCase( "rollback.idempotent", "rollback", @@ -810,12 +650,6 @@ def _case( _expired, ), ), - _case( - "maintain.naive-cutoff", - "maintain", - "a cutoff without a zone is refused", - _maintain_refuses_naive, - ), # existing lake KitCase( "open-existing-lake.idempotent", @@ -826,12 +660,6 @@ def _case( lambda c: (_attaching.of(c), _existing(c))[1], ), ), - _case( - "open-existing-lake.guarded", - "open_existing_lake", - "encryption and a missing protected set are refused", - _existing_refusals, - ), KitCase( "close-existing-lake.idempotent", "close_existing_lake", @@ -847,30 +675,13 @@ def _case( "latest, verify, committed-at, columns, views and a read, exact", _existing_reads, ), - _existing_forged("existing_latest"), - _existing_forged("existing_verify"), - _existing_forged("existing_committed_at"), - _existing_forged("existing_table_columns"), - _existing_forged("existing_views"), - _existing_forged("start_existing_read"), # clickhouse - _case( - "connect-clickhouse.guarded", - "connect_clickhouse", - "plain HTTP to another machine and a missing secret are refused", - _clickhouse_refusals, - ), _case( "close-clickhouse.harmless", "close_clickhouse", "closing what is not held changes nothing", _clickhouse_close, ), - _clickhouse_forged("clickhouse_columns", "t"), - _clickhouse_forged("clickhouse_catalog"), - _clickhouse_forged("clickhouse_estimate", "SELECT 1", ()), - _clickhouse_forged("clickhouse_now"), - _clickhouse_forged("start_clickhouse_read", "SELECT 1", (), (("a", "INTEGER"),), None), # encoding _case( "json-frame.columns", @@ -906,4 +717,5 @@ def _case( ), ) -STORE_KIT = Kit(contract=STORE, recorded=58, cases=CASES) +RECORDED = 153 +STORE_KIT = Kit(contract=STORE, recorded=RECORDED, cases=(*CASES, *code_cases(STORE))) diff --git a/packages/contracts/src/orvex/contracts/store/kit_codes.py b/packages/contracts/src/orvex/contracts/store/kit_codes.py new file mode 100644 index 00000000..69a57978 --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/kit_codes.py @@ -0,0 +1,756 @@ +"""The cases of the store's kit that prove every error code each operation declares: one case per +(operation, code), which makes that operation refuse with that code. The kit is refused when it +is built if a declared code has no such case, or a case proves an undeclared one +(KIT_CODE_UNPROVEN): no code is declared that the kit does not prove. +""" + +from __future__ import annotations + +import datetime as dt +import functools +import os +import sqlite3 +import threading +from collections.abc import Callable + +import pyarrow as pa +from orvex.contracts.interface import InterfaceContract +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.kit import KitCase +from orvex.contracts.kit_client import KitClient +from orvex.contracts.store.kit_services import CLICKHOUSE_PASSWORD, SAFE, clickhouse +from orvex.contracts.store.kit_support import ( + EXISTING_LIMITS, + LIMITS, + commit, + existing_lake, + folder, + local_location, + open_lake, + read_request, + refused, + sqlite_source, + statement, +) +from orvex.contracts.store.v1 import ( + ChangesRequest, + Parameter, + PostgresLakeLocation, + SqliteLakeLocation, + StatementRequest, + StatisticsRequest, + TablesRequest, +) +from orvex.contracts.store.values import ( + Budget, + ClickHouseHandle, + ClickHouseTarget, + ExistingLakeHandle, + ExistingLakeLimits, + LakeHandle, + Limits, + ReadHandle, + TransactionHandle, +) + +type Provoke = Callable[[KitClient], Callable[[], object]] + +HEAVY = "SELECT count(*) AS n FROM range(4000000000) AS a" +BUDGET = Budget(timeout_ms=30_000, memory_bytes=1_000_000_000) +UNSET = "STORE_KIT_UNSET_PASSWORD" # never set +SET = "STORE_KIT_SET_PASSWORD" # set to a value no server accepts +FORGED = { + "lake": LakeHandle("lake-forged"), + "read": ReadHandle("read-forged", pa.table({})), + "transaction": TransactionHandle("transaction-forged", None), + "existing": ExistingLakeHandle("existing-lake-forged", "forged"), + "clickhouse": ClickHouseHandle("clickhouse-forged", "db", ""), +} + + +def _now() -> dt.datetime: + return dt.datetime.now(dt.UTC) + + +def _bad_parameter(sql: str = "SELECT $p::BIGINT AS p") -> tuple[str, tuple[Parameter, ...]]: + return sql, (Parameter.model_validate({"name": "p", "type": "BIGINT", "value": "x"}),) + + +def _snapshot(client: KitClient, lake: LakeHandle) -> int: + found: int = client.current_snapshot(lake).id + return found + + +def _postgres(**changes: str) -> PostgresLakeLocation: + """A PostgreSQL catalog on a port nothing listens on, by default with a set password.""" + os.environ[SET] = "kit-not-a-password" + found = { + "kind": "lake_location", + "catalog": "postgres", + "connection": "host=127.0.0.1 port=1 dbname=kit user=kit", + "password_env": SET, + "data_path": str(folder()) + "/", + **changes, + } + return PostgresLakeLocation.model_validate(found) + + +# --- what a lake case starts from ---------------------------------------------------------------- + + +def _lake(client: KitClient) -> LakeHandle: + lake = open_lake(client, (sqlite_source(),)) + commit(client, lake, "CREATE TABLE t (n BIGINT)", "INSERT INTO t VALUES (1), (2)") + return lake + + +def _expired(client: KitClient) -> tuple[LakeHandle, int]: + """A lake whose first snapshot maintenance expired.""" + lake = open_lake(client) + first = commit(client, lake, "CREATE TABLE t AS SELECT 1 AS n").id + commit(client, lake, "INSERT INTO t VALUES (2)") + client.maintain(lake, _now(), _now()) + return lake, first + + +def _full(client: KitClient) -> LakeHandle: + """A lake of one slot and no queue, whose slot a read holds.""" + lake = client.open_lake(local_location(), Limits("256MB", 1, 1, 0, 60_000), ()) + snapshot = commit(client, lake, "CREATE TABLE t AS SELECT 1 AS n").id + client.start_read(lake, read_request("SELECT * FROM t", snapshot)) + found: LakeHandle = lake + return found + + +def _transaction(client: KitClient) -> TransactionHandle: + found: TransactionHandle = client.begin(_lake(client)) + return found + + +def _cancelled(client: KitClient) -> TransactionHandle: + transaction = _transaction(client) + client.cancel_transaction(transaction) + return transaction + + +def _timed(sql: str, timeout_ms: int = 50) -> StatementRequest: + return StatementRequest(kind="statement", sql=sql, parameters=[], timeout_ms=timeout_ms) + + +def _forged(kind: str, operation: str, *arguments: object) -> Provoke: + return lambda client: functools.partial( + client.send, operation, client.envelope(operation), FORGED[kind], *arguments + ) + + +# --- lake ---------------------------------------------------------------------------------------- + + +def _source_conflict(client: KitClient) -> Callable[[], object]: + source = sqlite_source() + return lambda: client.open_lake(local_location(), LIMITS, (source, source)) + + +def _missing_source(client: KitClient) -> Callable[[], object]: + source = sqlite_source().model_copy(update={"path": "/nonexistent/kit.sqlite"}) + return lambda: client.open_lake(local_location(), LIMITS, (source,)) + + +def _remote_data_without_storage(client: KitClient) -> Callable[[], object]: + root = folder() + location = SqliteLakeLocation( + kind="lake_location", + catalog="sqlite", + path=str(root / "catalog.sqlite"), + data_path="s3://kit-bucket/lake/", + ) + return lambda: client.open_lake(location, LIMITS, ()) + + +def _describe_refusal(client: KitClient) -> Callable[[], object]: + lake = _lake(client) + return lambda: client.describe(lake, "DELETE FROM t", ()) + + +def _describe_failure(client: KitClient) -> Callable[[], object]: + lake = _lake(client) + return lambda: client.describe(lake, "SELECT nothing_here FROM t", ()) + + +def _describe_parameter(client: KitClient) -> Callable[[], object]: + lake = _lake(client) + sql, parameters = _bad_parameter() + return lambda: client.describe(lake, sql, parameters) + + +def _source_refusal(sql: str, name: str = "field") -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + lake = _lake(client) + return lambda: client.describe_source(lake, name, sql, ()) + + return provoke + + +def _statistics_at(table: str, which: str) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + if which == "expired": + lake, snapshot = _expired(client) + else: + lake = _lake(client) + snapshot = 999_999 if which == "unknown" else _snapshot(client, lake) + request = StatisticsRequest.model_validate( + {"kind": "statistics", "tables": [table], "snapshot": snapshot} + ) + return lambda: client.statistics(lake, request) + + return provoke + + +def _changes_at(which: str) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + if which == "expired": + lake, since = _expired(client) + snapshot = _snapshot(client, lake) + else: + lake = _lake(client) + snapshot = _snapshot(client, lake) + since = {"unknown": 999_999, "backwards": snapshot}[which] + snapshot = {"unknown": snapshot, "backwards": snapshot - 1}[which] + request = ChangesRequest.model_validate( + {"kind": "changes", "tables": ["main.t"], "since": since, "snapshot": snapshot} + ) + return lambda: client.changes(lake, request) + + return provoke + + +def _columns_at(which: str) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + if which == "expired": + lake, snapshot = _expired(client) + else: + lake = _lake(client) + snapshot = 999_999 + request = TablesRequest(kind="tables", snapshot=snapshot) + return lambda: client.table_columns(lake, request) + + return provoke + + +def _busy(operation: str) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + lake = _full(client) + snapshot = _snapshot(client, lake) + arguments: dict[str, tuple[object, ...]] = { + "statistics": ( + StatisticsRequest.model_validate( + {"kind": "statistics", "tables": ["main.t"], "snapshot": snapshot} + ), + ), + "changes": ( + ChangesRequest.model_validate( + { + "kind": "changes", + "tables": ["main.t"], + "since": snapshot, + "snapshot": snapshot, + } + ), + ), + "table_columns": (TablesRequest(kind="tables", snapshot=snapshot),), + "start_read": (read_request("SELECT * FROM t", snapshot),), + "begin": (), + "maintain": (_now(), _now()), + } + envelope = client.envelope(operation) + return lambda: client.send(operation, envelope, lake, *arguments[operation]) + + return provoke + + +# --- read ---------------------------------------------------------------------------------------- + + +def _read_of(sql: str, snapshot: str = "current") -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + if snapshot == "expired": + lake, at = _expired(client) + else: + lake = _lake(client) + at = 999_999 if snapshot == "unknown" else _snapshot(client, lake) + return lambda: client.start_read(lake, read_request(sql, at)) + + return provoke + + +def _batch_of(sql: str, *, timeout_ms: int | None = None, cancel: bool = False) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + lake = _lake(client) + request = read_request(sql, _snapshot(client, lake)) + read = client.start_read(lake, request.model_copy(update={"timeout_ms": timeout_ms})) + if cancel: + client.cancel_read(read) + return lambda: client.read_batch(read) + + return provoke + + +# --- write --------------------------------------------------------------------------------------- + + +def _nested(client: KitClient) -> Callable[[], object]: + lake = _lake(client) + client.begin(lake) + return lambda: client.begin(lake) + + +def _writer_ahead(client: KitClient) -> Callable[[], object]: + lake = client.open_lake(local_location(), Limits("256MB", 1, 2, 2, 300), ()) + client.begin(lake) + + def from_another_thread() -> object: + outcome: list[object] = [] + thread = threading.Thread(target=lambda: outcome.append(_caught(client, lake))) + thread.start() + thread.join() + if isinstance(outcome[0], BaseException): + raise outcome[0] + return outcome[0] + + return from_another_thread + + +def _caught(client: KitClient, lake: LakeHandle) -> object: + try: + return client.begin(lake) + except BaseException as error: # noqa: BLE001 - handed back to the case's own thread + return error + + +def _in_transaction(operation: str, *arguments: object, state: str = "open") -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + transaction = _cancelled(client) if state == "cancelled" else _transaction(client) + envelope = client.envelope(operation) + return lambda: client.send(operation, envelope, transaction, *arguments) + + return provoke + + +def _failed_then_commit(client: KitClient) -> Callable[[], object]: + transaction = _transaction(client) + refused( + "STORE_QUERY_FAILED", + lambda: client.execute(transaction, statement("INSERT INTO t VALUES ('x')")), + ) + return lambda: client.commit(transaction) + + +def _timed_out_then_commit(client: KitClient) -> Callable[[], object]: + transaction = _transaction(client) + refused("STORE_TIMEOUT", lambda: client.execute(transaction, _timed(f"INSERT INTO t {HEAVY}"))) + return lambda: client.commit(transaction) + + +def _cancelled_commit(client: KitClient) -> Callable[[], object]: + transaction = _cancelled(client) + return lambda: client.commit(transaction) + + +def _conflict(client: KitClient) -> Callable[[], object]: + """Two writers of one SQLite catalog change the same table.""" + root = folder() + location = SqliteLakeLocation( + kind="lake_location", + catalog="sqlite", + path=str(root / "catalog.sqlite"), + data_path=(root / "data").as_posix() + "/", + ) + first = client.open_lake(location, LIMITS, ()) + commit(client, first, "CREATE TABLE t (n INTEGER)") + second = client.open_lake(location, LIMITS, ()) + mine = client.begin(first) + theirs = client.begin(second) + client.execute(mine, statement("INSERT INTO t VALUES (1)")) + client.execute(theirs, statement("DROP TABLE t")) + return lambda: client.commit(mine) + + +def _load(table: pa.Table, columns: tuple[tuple[str, str], ...], timeout_ms: int | None) -> Provoke: + return _in_transaction("load_arrow", "main", "loaded", table, columns, timeout_ms) + + +# --- existing lake ------------------------------------------------------------------------------- + + +def _attach(change: Callable[[KitClient], object]) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + made = change(client) + attachment, limits = made if isinstance(made, tuple) else (made, EXISTING_LIMITS) + return lambda: client.open_existing_lake(attachment, limits) + + return provoke + + +def _attachment(client: KitClient) -> object: + lake, attachment = existing_lake(client) + client.close_existing_lake(lake) + return attachment + + +def _encrypted(client: KitClient) -> object: + attachment = _attachment(client) + location = attachment.location.model_copy(update={"encrypted": True}) # type: ignore[attr-defined] + return attachment.model_copy(update={"location": location}) # type: ignore[attr-defined] + + +def _unprotected(client: KitClient) -> object: + return _attachment(client).model_copy(update={"protected_locations": None}) # type: ignore[attr-defined] + + +def _no_threads(client: KitClient) -> object: + return _attachment(client), ExistingLakeLimits(memory_limit="256MB", threads=0, timeout_ms=1) + + +def _untested_catalog(client: KitClient) -> object: + attachment = _attachment(client) + connection = sqlite3.connect(attachment.location.path) # type: ignore[attr-defined] + try: + connection.execute("UPDATE ducklake_metadata SET value = '9.9' WHERE key = 'version'") + connection.commit() + finally: + connection.close() + return attachment + + +def _on_postgres(**changes: str) -> Callable[[KitClient], object]: + def change(client: KitClient) -> object: + return _attachment(client).model_copy(update={"location": _postgres(**changes)}) # type: ignore[attr-defined] + + return change + + +def _existing(operation: str, *, snapshot: str = "latest", sql: str | None = None) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + lake, attachment = existing_lake(client) + latest = client.existing_latest(lake) + moved = latest.model_copy( + update={"committed_at": latest.committed_at - dt.timedelta(days=1)} + ) + at = moved if snapshot == "moved" else latest + arguments: dict[str, tuple[object, ...]] = { + "existing_verify": (at, attachment.model_copy(update={"name": "other"})), + "existing_table_columns": (at,), + "start_existing_read": ( + read_request(sql or "SELECT amount FROM readings", at.id), + at.committed_at, + ), + } + envelope = client.envelope(operation) + return lambda: client.send(operation, envelope, lake, *arguments[operation]) + + return provoke + + +# --- clickhouse ---------------------------------------------------------------------------------- + + +def _connect(target: Callable[[], ClickHouseTarget]) -> Provoke: + return lambda client: lambda: client.connect_clickhouse(target(), BUDGET) + + +def _plain_remote() -> ClickHouseTarget: + return ClickHouseTarget("clickhouse.example", 8123, "db", "kit", UNSET, tls=False) + + +def _secret_unset() -> ClickHouseTarget: + return ClickHouseTarget("localhost", 8123, "db", "kit", UNSET, tls=False) + + +def _nothing_listens() -> ClickHouseTarget: + os.environ[SET] = "kit-not-a-password" + return ClickHouseTarget("127.0.0.1", 1, "db", "kit", SET, tls=False) + + +def _administrator() -> ClickHouseTarget: + server = clickhouse() + return server.target(server.user, CLICKHOUSE_PASSWORD) + + +def _unsafe_reader() -> ClickHouseTarget: + server = clickhouse() + return server.reader(server.database(), {**SAFE, "join_use_nulls": "0"}) + + +def _engine(operation: str, *arguments: object) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + server = clickhouse() + database = server.database( + "readings (id UInt32, amount Decimal(18, 2)) ENGINE = MergeTree ORDER BY id", + "points (id UInt32, at Array(String)) ENGINE = MergeTree ORDER BY id", + ) + engine = client.connect_clickhouse(server.reader(database), BUDGET) + envelope = client.envelope(operation) + return lambda: client.send(operation, envelope, engine, *arguments) + + return provoke + + +# --- encoding and separation --------------------------------------------------------------------- + + +def _untyped(operation: str) -> Provoke: + table = pa.table({"x": pa.array([1], pa.int64())}) + arguments = {"json_frame": (table.schema.empty_table(),), "json_rows": (table, True)} + return lambda client: functools.partial( + client.send, operation, client.envelope(operation), *arguments[operation] + ) + + +def _separation(operation: str) -> Provoke: + def provoke(client: KitClient) -> Callable[[], object]: + root = str(local_location().path) + arguments: dict[str, tuple[object, ...]] = { + "location_components": ("c", "relative/path"), + "locations_overlap": ("c", "s3://bucket/../key", "s3://bucket/key"), + "check_apart": ("c", root + "/data", None, (root,), ()), + } + envelope = client.envelope(operation) + return lambda: client.send(operation, envelope, *arguments[operation]) + + return provoke + + +PARAMETER = "STORE_PARAMETER_INVALID" +SNAPSHOT_GONE = "STORE_SNAPSHOT_NOT_FOUND" +EXPIRED = "STORE_SNAPSHOT_EXPIRED" +STATEMENT = "STORE_STATEMENT_INVALID" +FAILED = "STORE_QUERY_FAILED" +SOURCE = "STORE_SOURCE_INVALID" +CANCELLED = "STORE_CANCELLED" +TIMEOUT = "STORE_TIMEOUT" +BUSY = "STORE_BUSY" +ONE = pa.table({"n": pa.array([1], pa.int64())}) +CAST = pa.table({"n": pa.array(["x"], pa.string())}) +MANY = pa.table({"n": pa.array(range(3_000_000), pa.int64())}) + +PROOFS: dict[tuple[str, str], Provoke] = { + # lake + ("open_lake", "STORE_LIMITS_REQUIRED"): lambda c: ( + lambda: c.open_lake(local_location(), Limits(None, None), ()) + ), + ("open_lake", "STORE_LOCATION_INVALID"): _remote_data_without_storage, + ("open_lake", "STORE_SECRET_MISSING"): lambda c: ( + lambda: c.open_lake(_postgres(password_env=UNSET), LIMITS, ()) + ), + ("open_lake", "STORE_SECRET_INLINE"): lambda c: ( + lambda: c.open_lake( + _postgres(connection="host=127.0.0.1 port=1 dbname=kit user=kit password=p"), LIMITS, () + ) + ), + ("open_lake", "STORE_TLS_REQUIRED"): lambda c: ( + lambda: c.open_lake( + _postgres(connection="host=db.example.com port=5432 dbname=kit user=kit"), LIMITS, () + ) + ), + ("open_lake", "STORE_CATALOG_UNAVAILABLE"): lambda c: ( + lambda: c.open_lake(_postgres(), LIMITS, ()) + ), + ("open_lake", "SOURCE_NOT_FOUND"): _missing_source, + ("open_lake", SOURCE): _source_conflict, + ("close_lake", PARAMETER): _forged("lake", "close_lake"), + ("current_snapshot", PARAMETER): _forged("lake", "current_snapshot"), + ("snapshot", PARAMETER): _forged("lake", "snapshot", 1), + ("tables", PARAMETER): _forged("lake", "tables"), + ("describe", PARAMETER): _describe_parameter, + ("describe", STATEMENT): _describe_refusal, + ("describe", FAILED): _describe_failure, + ("describe_source", PARAMETER): _forged("lake", "describe_source", "field", "SELECT 1", ()), + ("describe_source", STATEMENT): _source_refusal("DELETE FROM meters"), + ("describe_source", FAILED): _source_refusal("SELECT nothing_here FROM meters"), + ("describe_source", SOURCE): _source_refusal("SELECT 1", name="nope"), + ("statistics", PARAMETER): _forged( + "lake", + "statistics", + StatisticsRequest.model_validate( + {"kind": "statistics", "tables": ["main.t"], "snapshot": 1} + ), + ), + ("statistics", SNAPSHOT_GONE): _statistics_at("main.t", "unknown"), + ("statistics", EXPIRED): _statistics_at("main.t", "expired"), + ("statistics", BUSY): _busy("statistics"), + ("statistics", STATEMENT): _statistics_at("main.nope", "current"), + ("changes", PARAMETER): _forged( + "lake", + "changes", + ChangesRequest.model_validate( + {"kind": "changes", "tables": ["main.t"], "since": 1, "snapshot": 1} + ), + ), + ("changes", SNAPSHOT_GONE): _changes_at("unknown"), + ("changes", EXPIRED): _changes_at("expired"), + ("changes", BUSY): _busy("changes"), + ("changes", STATEMENT): _changes_at("backwards"), + ("table_columns", PARAMETER): _forged( + "lake", "table_columns", TablesRequest(kind="tables", snapshot=1) + ), + ("table_columns", SNAPSHOT_GONE): _columns_at("unknown"), + ("table_columns", EXPIRED): _columns_at("expired"), + ("table_columns", BUSY): _busy("table_columns"), + # read + ("start_read", PARAMETER): _forged("lake", "start_read", read_request("SELECT 1", 1)), + ("start_read", SNAPSHOT_GONE): _read_of("SELECT * FROM t", "unknown"), + ("start_read", EXPIRED): _read_of("SELECT * FROM t", "expired"), + ("start_read", BUSY): _busy("start_read"), + ("start_read", STATEMENT): _read_of("SELECT * FROM read_csv('/etc/passwd')"), + ("read_batch", PARAMETER): _forged("read", "read_batch"), + ("read_batch", FAILED): _batch_of( + "SELECT CAST(v AS INTEGER) AS n FROM (VALUES ('1'), ('x')) AS s(v)" + ), + ("read_batch", TIMEOUT): _batch_of(HEAVY, timeout_ms=50), + ("read_batch", CANCELLED): _batch_of(HEAVY, cancel=True), + ("cancel_read", PARAMETER): _forged("read", "cancel_read"), + ("close_read", PARAMETER): _forged("read", "close_read"), + # write + ("begin", PARAMETER): _forged("lake", "begin"), + ("begin", BUSY): _busy("begin"), + ("begin", STATEMENT): _nested, + ("begin", TIMEOUT): _writer_ahead, + ("execute", PARAMETER): _in_transaction( + "execute", + StatementRequest( + kind="statement", + sql=_bad_parameter("INSERT INTO t VALUES ($p)")[0], + parameters=list(_bad_parameter()[1]), + ), + ), + ("execute", STATEMENT): _in_transaction("execute", statement("ATTACH '/tmp/kit.db' AS other")), + ("execute", FAILED): _in_transaction("execute", statement("INSERT INTO t VALUES ('x')")), + ("execute", CANCELLED): _in_transaction( + "execute", statement("INSERT INTO t VALUES (3)"), state="cancelled" + ), + ("execute", TIMEOUT): _in_transaction("execute", _timed(f"INSERT INTO t {HEAVY}")), + ("value", PARAMETER): _forged("transaction", "value", statement("SELECT 1")), + ("value", STATEMENT): _in_transaction("value", statement("DELETE FROM t")), + ("value", FAILED): _in_transaction("value", statement("SELECT 'x'::INTEGER")), + ("value", CANCELLED): _in_transaction("value", statement("SELECT 1"), state="cancelled"), + ("value", TIMEOUT): _in_transaction("value", _timed(HEAVY)), + ("land", PARAMETER): _forged("transaction", "land", "field", "SELECT 1", (), "main", "l", None), + ("land", SOURCE): _in_transaction("land", "nope", "SELECT 1", (), "main", "l", None), + ("land", FAILED): _in_transaction( + "land", "field", "SELECT * FROM nothing_here", (), "main", "l", None + ), + ("land", CANCELLED): _in_transaction( + "land", "field", "SELECT * FROM meters", (), "main", "l", None, state="cancelled" + ), + ("land", TIMEOUT): _in_transaction( + "land", + "field", + "WITH RECURSIVE r(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM r WHERE i < 100000000) " + "SELECT count(*) AS n FROM r", + (), + "main", + "l", + 50, + ), + ("load_arrow", PARAMETER): _forged( + "transaction", "load_arrow", "main", "loaded", ONE, (("n", "BIGINT"),), None + ), + ("load_arrow", FAILED): _load(CAST, (("n", "INTEGER"),), None), + ("load_arrow", CANCELLED): _in_transaction( + "load_arrow", "main", "loaded", ONE, (("n", "BIGINT"),), None, state="cancelled" + ), + ("load_arrow", TIMEOUT): _load(MANY, (("n", "BIGINT"),), 1), + ("cancel_transaction", PARAMETER): _forged("transaction", "cancel_transaction"), + ("commit", PARAMETER): _forged("transaction", "commit"), + ("commit", CANCELLED): _cancelled_commit, + ("commit", FAILED): _failed_then_commit, + ("commit", TIMEOUT): _timed_out_then_commit, + ("commit", "STORE_TRANSACTION_CONFLICT"): _conflict, + ("rollback", PARAMETER): _forged("transaction", "rollback"), + # maintenance + ("maintain", PARAMETER): _forged("lake", "maintain", _now(), _now()), + ("maintain", BUSY): _busy("maintain"), + # existing lake + ("open_existing_lake", "STORE_LIMITS_REQUIRED"): _attach(_no_threads), + ("open_existing_lake", "STORE_LOCATION_INVALID"): _attach(_encrypted), + ("open_existing_lake", "STORE_SECRET_MISSING"): _attach(_on_postgres(password_env=UNSET)), + ("open_existing_lake", "STORE_TLS_REQUIRED"): _attach( + _on_postgres(connection="host=db.example.com port=5432 dbname=kit user=kit") + ), + ("open_existing_lake", "STORE_CONNECTION_FAILED"): _attach(_on_postgres()), + ("open_existing_lake", SOURCE): _attach(_unprotected), + ("open_existing_lake", "ENGINE_VERSION_UNTESTED"): _attach(_untested_catalog), + ("close_existing_lake", PARAMETER): _forged("existing", "close_existing_lake"), + ("existing_latest", PARAMETER): _forged("existing", "existing_latest"), + ("existing_verify", PARAMETER): _forged("existing", "existing_verify", None, None), + ("existing_verify", SOURCE): _existing("existing_verify"), + ("existing_committed_at", PARAMETER): _forged("existing", "existing_committed_at", 1), + ("existing_table_columns", PARAMETER): _forged("existing", "existing_table_columns", None), + ("existing_table_columns", SOURCE): _existing("existing_table_columns", snapshot="moved"), + ("existing_views", PARAMETER): _forged("existing", "existing_views", None), + ("start_existing_read", PARAMETER): _forged( + "existing", "start_existing_read", read_request("SELECT 1", 1), _now() + ), + ("start_existing_read", STATEMENT): _existing( + "start_existing_read", sql="SELECT * FROM read_csv('/etc/passwd')" + ), + ("start_existing_read", FAILED): _existing( + "start_existing_read", sql="SELECT nothing_here FROM readings" + ), + ("start_existing_read", SOURCE): _existing("start_existing_read", snapshot="moved"), + # clickhouse + ("connect_clickhouse", "STORE_TLS_REQUIRED"): _connect(_plain_remote), + ("connect_clickhouse", "STORE_SECRET_MISSING"): _connect(_secret_unset), + ("connect_clickhouse", "STORE_CONNECTION_FAILED"): _connect(_nothing_listens), + ("connect_clickhouse", "SOURCE_NOT_READ_ONLY"): _connect(_administrator), + ("connect_clickhouse", "ENGINE_SETTING_REQUIRED"): _connect(_unsafe_reader), + ("clickhouse_columns", PARAMETER): _forged("clickhouse", "clickhouse_columns", "t"), + ("clickhouse_columns", "SOURCE_NOT_FOUND"): _engine("clickhouse_columns", "nothing_here"), + ("clickhouse_columns", "UNSUPPORTED_SOURCE_TYPE"): _engine("clickhouse_columns", "points"), + ("clickhouse_catalog", PARAMETER): _forged("clickhouse", "clickhouse_catalog"), + ("clickhouse_estimate", PARAMETER): _forged( + "clickhouse", "clickhouse_estimate", "SELECT 1", () + ), + ("clickhouse_estimate", STATEMENT): _engine("clickhouse_estimate", "SELECT 1 FORMAT JSON", ()), + ("clickhouse_now", PARAMETER): _forged("clickhouse", "clickhouse_now"), + ("start_clickhouse_read", PARAMETER): _forged( + "clickhouse", "start_clickhouse_read", "SELECT 1", (), (("a", "INTEGER"),), None + ), + ("start_clickhouse_read", STATEMENT): _engine( + "start_clickhouse_read", "SELECT 1 SETTINGS readonly = 0", (), (("a", "INTEGER"),), None + ), + ("start_clickhouse_read", FAILED): _engine( + "start_clickhouse_read", "SELECT nothing_here FROM readings", (), (("a", "INTEGER"),), None + ), + # encoding and separation + ("json_frame", "STORE_TYPE_UNSUPPORTED"): _untyped("json_frame"), + ("json_rows", "STORE_TYPE_UNSUPPORTED"): _untyped("json_rows"), + ("location_components", SOURCE): _separation("location_components"), + ("locations_overlap", SOURCE): _separation("locations_overlap"), + ("check_apart", SOURCE): _separation("check_apart"), +} + + +def _case(operation: str, code: str, provoke: Provoke) -> KitCase: + def check(client: KitClient) -> None: + refused(code, provoke(client)) + + identifier = f"{operation.replace('_', '-')}.raises.{code.lower().replace('_', '-')}" + return KitCase(identifier, operation, f"{operation} refuses with {code}", check) + + +def code_cases(contract: InterfaceContract) -> tuple[KitCase, ...]: + """One case per declared (operation, code); KIT_CODE_UNPROVEN when the two differ.""" + declared = {(o.name, code) for o in contract.operations for code in o.errors} + unproven = sorted(declared - set(PROOFS)) + undeclared = sorted(set(PROOFS) - declared) + if unproven or undeclared: + raise InterfaceContractError( + "KIT_CODE_UNPROVEN", + f"kit {contract.name}: declared codes without a case {unproven}; " + f"cases of undeclared codes {undeclared}", + {"contract": contract.name}, + ) + return tuple( + _case(operation, code, PROOFS[operation, code]) for operation, code in sorted(PROOFS) + ) diff --git a/packages/contracts/src/orvex/contracts/store/kit_services.py b/packages/contracts/src/orvex/contracts/store/kit_services.py new file mode 100644 index 00000000..86b0c2da --- /dev/null +++ b/packages/contracts/src/orvex/contracts/store/kit_services.py @@ -0,0 +1,77 @@ +"""The servers some cases of the store's kit need, taken from the environment as the store's own +tests take them: a ClickHouse administrator of a disposable server (`STORE_TEST_CLICKHOUSE`, +`host:port:user`, and `STORE_TEST_CLICKHOUSE_PASSWORD`). Where none is given, a case that needs +one fails (KIT_SERVICE_MISSING); it never skips. +""" + +from __future__ import annotations + +import os +import secrets +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass + +from orvex.contracts.store.interface import REQUIRED_SETTINGS +from orvex.contracts.store.values import ClickHouseTarget + +CLICKHOUSE = "STORE_TEST_CLICKHOUSE" +CLICKHOUSE_PASSWORD = "STORE_TEST_CLICKHOUSE_PASSWORD" +READER_PASSWORD = "STORE_KIT_CLICKHOUSE_READER_PASSWORD" # set for each reader, never stored +SAFE = { + "readonly": "1", + **REQUIRED_SETTINGS, + "max_execution_time": "30", + "max_memory_usage": "1000000000", +} + + +@dataclass(frozen=True, slots=True) +class ClickHouseServer: + host: str + port: int + user: str + password: str + + def run(self, sql: str) -> str: + request = urllib.request.Request( + f"http://{self.host}:{self.port}/", + data=sql.encode(), + headers={"X-ClickHouse-User": self.user, "X-ClickHouse-Key": self.password}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + body: bytes = response.read() + return body.decode() + + def target(self, user: str, password_env: str, database: str = "default") -> ClickHouseTarget: + return ClickHouseTarget(self.host, self.port, database, user, password_env, tls=False) + + def reader(self, database: str, settings: Mapping[str, str] = SAFE) -> ClickHouseTarget: + """A new user of `database` whose profile has `settings`, its password in the + environment.""" + name = f"kit_{secrets.token_hex(6)}" + password = secrets.token_hex(16) + listed = ", ".join(f"{key} = '{value}'" for key, value in settings.items()) + self.run(f"CREATE SETTINGS PROFILE {name} SETTINGS {listed}") + self.run(f"CREATE USER {name} IDENTIFIED BY '{password}' SETTINGS PROFILE {name}") + self.run(f"GRANT SELECT ON {database}.* TO {name}") + os.environ[READER_PASSWORD] = password + return self.target(name, READER_PASSWORD, database) + + def database(self, *tables: str) -> str: + """A new database with `tables` (`name (columns) ENGINE = …` each).""" + name = f"kit_{secrets.token_hex(6)}" + self.run(f"CREATE DATABASE {name}") + for table in tables: + self.run(f"CREATE TABLE {name}.{table}") + return name + + +def clickhouse() -> ClickHouseServer: + named = os.environ.get(CLICKHOUSE) + if not named: + raise AssertionError( + f"KIT_SERVICE_MISSING: this case needs a ClickHouse server ({CLICKHOUSE})" + ) + host, port, user = named.split(":") + return ClickHouseServer(host, int(port), user, os.environ.get(CLICKHOUSE_PASSWORD, "")) diff --git a/packages/contracts/tests/test_store_kit_codes.py b/packages/contracts/tests/test_store_kit_codes.py new file mode 100644 index 00000000..b046e0ef --- /dev/null +++ b/packages/contracts/tests/test_store_kit_codes.py @@ -0,0 +1,46 @@ +"""#486 G1: the store's kit proves every code each operation declares, and nothing else; a code +declared without its case, or a case of an undeclared code, refuses the kit.""" + +from __future__ import annotations + +import dataclasses + +import pytest +from orvex.contracts.interface_errors import InterfaceContractError +from orvex.contracts.store.interface import STORE +from orvex.contracts.store.kit import STORE_KIT +from orvex.contracts.store.kit_codes import code_cases + + +def _with_errors(operation: str, errors: tuple[str, ...]) -> object: + operations = tuple( + dataclasses.replace(o, errors=errors) if o.name == operation else o + for o in STORE.operations + ) + return dataclasses.replace(STORE, operations=operations) + + +def test_every_declared_code_has_its_case_in_the_kit() -> None: + declared = {(o.name, code) for o in STORE.operations for code in o.errors} + proven = { + (case.operation, case.id.split(".raises.")[1].upper().replace("-", "_")) + for case in STORE_KIT.cases + if ".raises." in case.id + } + assert proven == declared + + +def test_a_code_declared_without_its_case_refuses_the_kit() -> None: + widened = _with_errors("held", ("STORE_BUSY",)) + with pytest.raises(InterfaceContractError) as caught: + code_cases(widened) # type: ignore[arg-type] + assert caught.value.code == "KIT_CODE_UNPROVEN" + assert "('held', 'STORE_BUSY')" in caught.value.message + + +def test_a_case_of_a_code_no_longer_declared_refuses_the_kit() -> None: + narrowed = _with_errors("json_frame", ()) + with pytest.raises(InterfaceContractError) as caught: + code_cases(narrowed) # type: ignore[arg-type] + assert caught.value.code == "KIT_CODE_UNPROVEN" + assert "('json_frame', 'STORE_TYPE_UNSUPPORTED')" in caught.value.message diff --git a/packages/store/tests/test_contract_kit.py b/packages/store/tests/test_contract_kit.py index a0389c84..4dbc9a6f 100644 --- a/packages/store/tests/test_contract_kit.py +++ b/packages/store/tests/test_contract_kit.py @@ -80,7 +80,7 @@ def _fields(envelope: CallEnvelope) -> dict[str, object]: [ (_ExecutesTwice, "execute.idempotent"), (_DecimalAsFloat, "start-read.exact"), - (_ForgedHandleAccepted, "read-batch.ends"), + (_ForgedHandleAccepted, "read-batch.raises.store-parameter-invalid"), ], ) def test_the_kit_fails_a_planted_broken_guarantee(planted: type[Store], case: str) -> None: diff --git a/relative/x/catalog.ducklake b/relative/x/catalog.ducklake new file mode 100644 index 0000000000000000000000000000000000000000..5d30c39741d46f0a79858f6d2eae45ae13ad8af4 GIT binary patch literal 1847296 zcmeI*Ypf($T>$X9GxyH(&I}8zEU=3kRuf3f&a8kR&=06fqEU$eTQ2$*UJW_KvSVe*gLx$Iopl0RjXF5FkK+009C7 z2oNA}qXKJBzwoi2{g<7$&ebpA_~GvO`OeUh&ez+!;|&lXK!5-N0t5&UAV7cs0Rpow zu==Mz{*@1W@GE~-rL(>3Q;pBE$&deC|L>_8B0zuu0RjXF5FkK+009ECAh2-qQxCoO zUw`$N%e2h5vezwT2U^)&D?6~9W$Rgd&#yL@*Rsxufu%Gh{fmjKF);>Xj&oTtPrIFG z_ca==ou$WCnvdm;)qK0zYHl|h>96$rw77#K=O~tOs{8Au^_88~wRZd2Z1=e5VlgLL z+0o7%jpbGzBTly4=d)S)q7xT5&=&47xn`c+@y5%mG zt6bk~<(orGJzh>-Yp&*L;;Z|UR88D|a$|VnspYlh?d9f5qm`d)?yM9E9{BZ%azf)` zb7g1v_b1kNR#vte&6Sn)$Mb^YSpT|DY&19Xwe3bHeM9Mf`M|;(wJE%JaGq8qHecGE z&XmV#>6gpTT%H|ix8pI8PR>NkwA=e0L9f~sL5Vp6`4mZ`rlp8F*vgKTv({z( z=%ARd0jrUCW^w8iuh>no-7FaD#wUV9t!(sW?ptv?_*nkrP;ptP3-lV*FQ1ya>dU*T zMdfyGc_k{hXS15o$@HF9w%DET`U30bYO!M}VU7BlYJQ`+ePPJ_Woc|laqL$fN8?JIs6+hitesA75`{Z`MHB<{7Kerw= z^=2c@GBj~tBRG{l?b-*X{wWZA`H;T!1p*F76A1{_174vHI ziJ?TDN{KqZxxTYeyUp$PXR-sagFN_bHuqeXeI}b52!wV!?MI(Yb>dL1`Agr>i|Ux` z{vUkj_x`|lzVk_WEY?eBQX2Z*B$MTm)+?`^J=(>|FNtl&UN{dGV3PnfR}qw=++KL@`gomd%UAO4yit|A+6k%TolO6F^VWsN=-Te)%HBXm55>)mmcB!y3SPOD`%&=rDm=Hg zfb7bzkwxS{o?SUSP()rgTtxb^{%$EG-`p)EsdQ{Lww5mBt(}#;)rioE#$^(aKE+3nQi-U5&%0x~5kgZpAKe#H2VZTHB6; z#I43!b1h}sV51>_VtcdMK}!<{4r~_F3Zu!9Khf+QP7M5XF|BjN)S3YKNGy8FM;i|e z21u_VQXr-n89&EFC2VwAqBxY#Iy?6B*|U*R zarl;yi%$%tq{gVKKwiGonUZI7=hAtAD4=h<*;5*wB}z%ZBa?oO&TyT{^I3K!d;izk z?Ne8)Hue4nrWFon%yo>Z%fqtPiKCaYvguYpvsx$Ii9rxGdQsC?nVA}5aiDrR4kXht zVdoRawX>Qhv$OA-LUc@+S%+j+6}jnqOy$#=lp?)b3*Eg#Rg%g&Tdj=CW zJC%x*q#`PJ_NvdOP?TyjVb9jv?Mj%My_J;*_+{)IbfDh}irPS)fp}3=W*`%8=MHzm z#>&ZA4dB~p$7HW2!tCVE?W*WBRyr{<+q*$=YBxB^DarX+t@3BKdMiuTOI3XJo%H(j zb6@@abmr?)XIivGZt-=#3EMV{vzn>1Q!7bHicxX0S9>q2q2jY}$6qTgOT^4J@AgdC z=na^#+ML;ap~`BwvYJBsV8XDmx9zV)XZ%9p6vAe3r#kf&sw{>pi`g-mdN@pdg`;=F zQ!f}3R^T&xTh8suyHtRRPv5a^bw|gg&Xl0NJ;$~SgHbd)C3|*Ka#w}T?1aOFwT%GY zv$;6+0IpeKnaJqQzpC(BSycxf%(H(wIQ0i7UOV{U#Kq?3ax)%L?><^HZuc46J0A7Da~{k|QPThPRm8}LrvE7a{oNQ@URzmS3-{u=*P?rd z;`ytwGo&bqhx$i9Jaeqm&qHl@4aLK0tIg`+*ksYy!$b|w7hW;-B6PCLv)WwRTu%=h z?HZK`7UI#Z^rT&EOifQ2M~a`;O^L^%W1;axS50RYH}iCCjomN2YU*~qE)Yt9)s2+| zSU0{@98Gn2IeuMdc(}fwjEKvvA6prE7H*P`B%&Z*y?^RjuO5uFMm$g!VG&R37o~9Y z)FgK}ir~l~8!0#D`NxypwT}qS-t^jPdXK(T_U;X3cLXow*|j*IJLM-2@9E`9+0Raj z-{G+-58@f#$$Qy|Dr0MDy{4C%BgN?9IIlW7=yYZfP1qO+kHI}iAYCbXK$)*PgPET~ zXE676ky1U@-xW;#l!O(%pp5GhVoYQn-&)_FzPaX#OUehc!I)-Yl`ff;c- zrk9?@cn&t^==5`u>W;oAWoxki{hp~uWUn6Rrn7uvIz{H_sb&2-z88~8QVBIUbdTdb~-JmZ|ikWaI<}kRg11|UfpO1A<=a)YnG2!ZW?6!{dBf>Xs(XB1C60@tZinJfF=SxRA{~ z^|Iv{nYyF7ZT;26&S~GfA-k`;=P7Z@Zgz?m)5wV_CgYf8uL3y!=%nwMr#$XIbgNHq z4xXo~IpJ`tS2*G6sKbK^o1L?mQj+$jgSj(>u(871b!8)Z3Yk9_wOQ1)=d)86VkjOx zidrH1f_y$Z_UGAsOPhIgBG-!E;Igc&8;jcNO7=JJpSzkpl;Y(*+1$H2hY{7{E~UlA zZlrH<<&5Q-&EhuE&YJV=-E}L!CszK*!;f^&+g82ZoqpnU>q_>)@4uS8v0P;9X4d`b z-j+w=V=q{2`K~@HVguu;`qb_m-oWm!Mr~lvqC4y4)l97U<&&=Y@`03uw9D?^R1`TG z2Xd3=WKn<57cmiKvcIpG`P3O*_x&>6T&W%Ce@$&CNmT)_(lcS^)+eWb@kOGB5^-qP znBFh?lGDwNx(Y4~>lAh^cj7#l~cWFOD3X3-({O|JXW$bjwzRScz1=*G~^2Xr?l z_B&&bRC}pVj+4!!3Tjmaq`5kWlHY!#wOjeSj#us4>kDXEbJR~vOB+t(^(C=7M(X`) zOd5xZ^zhnmq0kM4!*F_f+jV0@bsEoS6BhoPP0qogG}k9*!usH5lQWQ-B5;P16Q`ca z(lk^{lx@Qq%GJuF+ODEBRozck1*1Cu7Z>m@m!G&$^-^r!z=$u$PGVrh*W*Iez=+Ss z{$OCl-xhC8xLkf>p`SgpYmTeY_58q3{va9-11BJ6_8$DntMJH$Zv9&2#EpIr#^d$2 z5^jA;#zt7Rk-h!>rIGvGp_+HU8j*kZS0Db;i$C*J1aG

5WLZ`?cDsZQoF}G~-*; zJ@GO_O_g3q)W%2^^ph0p`Kl0%{T?jVGbAw-O?z*9wO<*#!F>~FVcV;%s*;srUKW?%bgK zV7U*DIxDt)@TQAh9IfosTD+8^u@o;STvgw`#yy0?dD_)^B^t*Txq?q1$^GUi}Z`Uk07(z$#>=&bX zS{xpy>s_tvq0u2!ZP}Pmik9u&6-s?KUafC#$6IGMHX{qmcOs3Pm;&i={M$;u8LAHY zZnqk$XHFJB8U1ROW4#pYijcI%+Lvsm$}Od%wH)u;ORsYp`eEkW3(d7e9`*8be@4nHYkRSI{)s5ZQT9Y4N>w7`slv*#n2NFAvja3i(Rqp~DH$CN4 z+zvZBr^ei#exmnst$1%&;_kcgKVC4_n_Tnmts8rv*YWARu`6ny&%bW!wNLMTk)3?( zK5<$*p{#*M=iK<}GRHJpRxNUu^S|_?%7nk$LQ=L6HrmP4Kr>S*XM~ey5 zt*|@mtbGt+WD}(J`Ne1zr&qSeYsi019R2@s>^}!ad^8?x8W{2G_w5?-^O2VW zbNok@xz%lVaSb6>G{uYVQ5z0lg}t}$-r{r?pt{dE4PVK=@YMrXvv`kaanAJ7PHT8f zy2}~koI?8aP6@}2ir>>+dH34M(0w&=7IWl9{O+&Pl4HiYb@c9E?#_M7*KuLLwqvHw zkv^YI8W+`BwGrn@ulGb@l*^RCyDzg1W#b;JEl=!S&)40!+Dfr<8hh=quS$%j==%n) z9ma^S?jG@_fs2VT$7cpECdP=573b#q?nL%HQk4_McV9?$??j%ROOkbEa$m23TH!!+ zT<;&9Xc}AOt3|rgw2~;lXS*cc&N^Yjc!*jyTs(q-w&1X*DC>K+!yV`BKNEMaiGOmjf^u~vl&HY z$&SO|nJZy6*0qLv&UfcW=hU}wba9w8izeJlcApm>xo^7bE8m$bxK*7p)fZq-7*E}E zO5!Q)n>txlTV`)dE)J$->ebp5Rybjt&S-O?wwX{yN^d{hYY00!=-zPML+NnyGNw*dkK2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB=Dg7x<2!`lhGO|M2fzc_`Mjkk&STuvq8Cd~<7geQjwo zZ*J$U#@Q!Z&FyAmqq%({kC9Jq#K>a4ji-+m-_`&A@<=t*`F>-370X@dEccOzA9-|V z>9I$@>%rDr-n?}7Z2qR!xy^i~xxIWbKmEk%w3gH7nk!p*xtJe)^xcCCEdScQo-}%{ zc1|M+5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5;& z>lOIqM?d-EM_N0dn2#SGNIyO}*G^xn&E>Urd>%}Hl%MqXTsgcD-@E5O9n;@X%^ctT z^Y@K5NPqwV0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2+X*^3(qV(aP8bb+c_a=9~bIDO3(|B4YYSbU1{<*+XT1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAVAn0RjXF z5Fl`;3sh~8%bkJ4>1(d|SB!|k;!}(-hkX$sK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly WK!5-N0t5&UAV7cs0Rnfa!2bbMiI_hC literal 0 HcmV?d00001 From 5fbab76b6184ee6cda2c6464967b833fb3cdc9e4 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:03:45 -0400 Subject: [PATCH 20/21] fix(store): the store's face is its api, test dependencies keep their extras, CI installs the helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The structure check's store façade becomes `orvex.store.api`, which the implementation's own modules may not import back; a test dependency may name its extras (`moto[server]`), compared by its name; the Windows job installs `orvex-testing`; the store tests import the contract's types from the contract; and the test helpers' docstring names no product. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .../contracts/src/orvex/contracts/manifest.py | 19 +++++++++++++------ packages/contracts/tests/test_manifest.py | 14 +++++++++++++- packages/server/tests/test_existing_lakes.py | 3 ++- packages/store/src/orvex/store/module.toml | 2 +- packages/store/tests/test_clickhouse.py | 8 +++++--- packages/store/tests/test_contract_kit.py | 9 ++------- .../store/tests/test_existing_lake_reads.py | 4 ++-- tests/tools/test_structure.py | 6 +++--- tools/checks/structure.py | 5 +++-- 9 files changed, 44 insertions(+), 26 deletions(-) diff --git a/packages/contracts/src/orvex/contracts/manifest.py b/packages/contracts/src/orvex/contracts/manifest.py index e9ed2200..b2463534 100644 --- a/packages/contracts/src/orvex/contracts/manifest.py +++ b/packages/contracts/src/orvex/contracts/manifest.py @@ -35,6 +35,8 @@ # A module, or an alternative implementation, by its package path under the namespace. MODULE = re.compile(r"[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*") DISTRIBUTION = re.compile(r"[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?") +# A test dependency may name its extras (`moto[server]`): what its own requirements need. +WITH_EXTRAS = re.compile(rf"{DISTRIBUTION.pattern}(\[[a-z0-9]+(,[a-z0-9]+)*\])?") IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") @@ -63,8 +65,8 @@ class Manifest: def canonical(distribution: str) -> str: - """A distribution name in its normalized form (PEP 503).""" - return re.sub(r"[-_.]+", "-", distribution).lower() + """A distribution name in its normalized form (PEP 503), without its extras.""" + return re.sub(r"[-_.]+", "-", distribution.partition("[")[0]).lower() def parse_manifest(text: str, source: str) -> Manifest: @@ -102,11 +104,16 @@ def _strings( return tuple(value) -def _distributions(value: object, where: str) -> tuple[str, ...]: - names = [canonical(name) for name in _strings(value, where, DISTRIBUTION)] +def _distributions( + value: object, where: str, pattern: re.Pattern[str] = DISTRIBUTION +) -> tuple[str, ...]: + """The distributions listed, normalized; with `WITH_EXTRAS`, each keeps its extras.""" + listed = _strings(value, where, pattern) + names = [canonical(name) for name in listed] if len(set(names)) != len(names): raise ValueError(f"{where} names a distribution twice") - return tuple(sorted(names)) + extras = [name[name.index("[") :] if "[" in name else "" for name in listed] + return tuple(sorted(f"{n}{e}" for n, e in zip(names, extras, strict=True))) def _capabilities(value: object, where: str) -> tuple[str, ...]: @@ -160,7 +167,7 @@ def _manifest(data: dict[str, Any]) -> Manifest: tuple(provides), tuple(requires), _distributions(dependencies["runtime"], "dependencies.runtime"), - _distributions(dependencies["tests"], "dependencies.tests"), + _distributions(dependencies["tests"], "dependencies.tests", WITH_EXTRAS), ) diff --git a/packages/contracts/tests/test_manifest.py b/packages/contracts/tests/test_manifest.py index d52d46f5..cf1bf3f9 100644 --- a/packages/contracts/tests/test_manifest.py +++ b/packages/contracts/tests/test_manifest.py @@ -7,7 +7,7 @@ import orvex.contracts import pytest from orvex.contracts.interface_errors import InterfaceContractError -from orvex.contracts.manifest import Provided, Required, parse_manifest +from orvex.contracts.manifest import Provided, Required, canonical, parse_manifest from orvex.contracts.version import Version FULL = """ @@ -53,6 +53,13 @@ def test_provides_and_requires_may_be_absent() -> None: assert (manifest.provides, manifest.requires, manifest.runtime) == ((), (), ()) +def test_a_test_dependency_keeps_its_extras_and_is_compared_by_name() -> None: + text = FULL.replace('tests = ["pytest"]', 'tests = ["pytest", "Moto[server]"]') + manifest = parse_manifest(text, "store") + assert manifest.tests == ("moto[server]", "pytest") + assert canonical("Moto[server]") == "moto" + + INVALID = { "not toml": "module = ", "no module": "[dependencies]\nruntime = []\ntests = []\n", @@ -78,6 +85,11 @@ def test_provides_and_requires_may_be_absent() -> None: + '[[requires]]\ncontract = "security-context"\nversion = "1.0"\ncapabilities = ["read"]\n', "bad contract": FULL.replace('contract = "store"', 'contract = "Store"'), "not a string list": FULL.replace('tests = ["pytest"]', "tests = [1]"), + "extras of a runtime dependency": FULL.replace('"duckdb"', '"duckdb[all]"'), + "malformed extras": FULL.replace('tests = ["pytest"]', 'tests = ["moto[server"]'), + "a test dependency twice, once with extras": FULL.replace( + 'tests = ["pytest"]', 'tests = ["moto", "moto[server]"]' + ), } diff --git a/packages/server/tests/test_existing_lakes.py b/packages/server/tests/test_existing_lakes.py index 0489e420..2faa7165 100644 --- a/packages/server/tests/test_existing_lakes.py +++ b/packages/server/tests/test_existing_lakes.py @@ -19,6 +19,7 @@ import duckdb import pytest from orvex.access.local import LocalRequest +from orvex.contracts.store.client_sources import open_existing_lake from orvex.contracts.store.errors import StoreError from orvex.server import cli, existing_lakes, semantic_http from orvex.server.commands import project_lock @@ -154,7 +155,7 @@ def test_the_managed_lake_and_each_existing_lake_share_the_budget( monkeypatch.setattr(cli, "memory_limit", lambda: "900MB") monkeypatch.setattr(cli, "instance_directory", lambda identity: served.instance) given: list[str] = [] - real = existing_lakes.open_existing_lake + real = open_existing_lake def recorded(client: Any, attachment: Any, limits: Any) -> Any: # noqa: ANN401 given.append(limits.memory_limit) diff --git a/packages/store/src/orvex/store/module.toml b/packages/store/src/orvex/store/module.toml index 0cd22eed..02747dd8 100644 --- a/packages/store/src/orvex/store/module.toml +++ b/packages/store/src/orvex/store/module.toml @@ -4,7 +4,7 @@ module = "store" [dependencies] runtime = ["duckdb", "orvex-contracts", "pyarrow"] -tests = ["hypothesis", "moto", "orvex-testing", "pytest"] +tests = ["hypothesis", "moto[server]", "orvex-testing", "pytest"] [[provides]] contract = "store" diff --git a/packages/store/tests/test_clickhouse.py b/packages/store/tests/test_clickhouse.py index f11215d6..90e2e0c5 100644 --- a/packages/store/tests/test_clickhouse.py +++ b/packages/store/tests/test_clickhouse.py @@ -19,14 +19,16 @@ import pyarrow as pa import pytest from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.interface import REQUIRED_SETTINGS from orvex.contracts.store.v1 import Parameter +from orvex.contracts.store.values import Budget, ClickHouseTarget from orvex.store import clickhouse -from orvex.store.clickhouse import Budget, ClickHouse, ClickHouseTarget, duckdb_type +from orvex.store.clickhouse import ClickHouse, duckdb_type BUDGET = Budget(timeout_ms=30_000, memory_bytes=2_000_000_000) SAFE = { "readonly": "1", - **clickhouse.REQUIRED_SETTINGS, + **REQUIRED_SETTINGS, "max_execution_time": "30", "max_memory_usage": "1000000000", } @@ -134,7 +136,7 @@ def test_a_profile_that_is_not_read_only_or_can_change_settings_is_refused( def test_every_required_setting_has_a_wrong_value_case() -> None: - assert set(WRONG) == set(clickhouse.REQUIRED_SETTINGS) + assert set(WRONG) == set(REQUIRED_SETTINGS) @pytest.mark.parametrize(("name", "value"), sorted(WRONG.items())) diff --git a/packages/store/tests/test_contract_kit.py b/packages/store/tests/test_contract_kit.py index 4dbc9a6f..f0d4f3aa 100644 --- a/packages/store/tests/test_contract_kit.py +++ b/packages/store/tests/test_contract_kit.py @@ -3,6 +3,7 @@ from __future__ import annotations +import dataclasses from importlib.resources import files import pyarrow as pa @@ -43,9 +44,7 @@ def execute( self, envelope: CallEnvelope, transaction: TransactionHandle, request: StatementRequest ) -> None: super().execute(envelope, transaction, request) - super().execute( - CallEnvelope(**{**_fields(envelope), "idempotency_key": None}), transaction, request - ) + super().execute(dataclasses.replace(envelope, idempotency_key=None), transaction, request) class _DecimalAsFloat(Store): @@ -71,10 +70,6 @@ def read_batch(self, envelope: CallEnvelope, read: ReadHandle) -> pa.Table | Non return super().read_batch(envelope, read) -def _fields(envelope: CallEnvelope) -> dict[str, object]: - return {name: getattr(envelope, name) for name in CallEnvelope.__slots__} - - @pytest.mark.parametrize( ("planted", "case"), [ diff --git a/packages/store/tests/test_existing_lake_reads.py b/packages/store/tests/test_existing_lake_reads.py index d26f14b2..cb19b63d 100644 --- a/packages/store/tests/test_existing_lake_reads.py +++ b/packages/store/tests/test_existing_lake_reads.py @@ -13,7 +13,7 @@ import duckdb import pytest -from orvex.contracts.store.errors import StoreError +from orvex.contracts.store.errors import FILE_REFUSED, SNAPSHOT_CHANGED, StoreError from orvex.contracts.store.v1 import ( AzureStorage, ExistingLakeAttachment, @@ -28,7 +28,7 @@ StatementRequest, ) from orvex.store.existing_lake import ExistingLake -from orvex.store.existing_lake_reads import FILE_REFUSED, SNAPSHOT_CHANGED, LakeReads +from orvex.store.existing_lake_reads import LakeReads from orvex.store.lake import Limits, open_lake from .test_existing_lake import MEMORY, Catalog diff --git a/tests/tools/test_structure.py b/tests/tools/test_structure.py index 413be402..28864306 100644 --- a/tests/tools/test_structure.py +++ b/tests/tools/test_structure.py @@ -160,7 +160,7 @@ def test_a_cycle_through_typing_only_imports_is_not_reported(tmp_path: Path) -> def test_an_internal_module_importing_the_facade_fails(tmp_path: Path) -> None: - root = _package(tmp_path, {"__init__.py": "", "lake.py": "", "x.py": "from . import lake\n"}) + root = _package(tmp_path, {"__init__.py": "", "api.py": "", "x.py": "from . import api\n"}) assert _codes(structure.imports(root)) == [("error", "FACADE_IMPORTED")] @@ -172,10 +172,10 @@ def test_a_listed_cycle_or_facade_import_passes_and_a_stale_one_fails(tmp_path: ) files = { "__init__.py": "", - "lake.py": "", + "api.py": "", "a.py": "from . import b\n", "b.py": "from . import a\n", - "x.py": "from . import lake\n", + "x.py": "from . import api\n", } found = structure.imports(_package(tmp_path, files, exceptions)) assert sorted((f.code, f.path) for f in found) == [ diff --git a/tools/checks/structure.py b/tools/checks/structure.py index 29eba8d2..4fb667f2 100644 --- a/tools/checks/structure.py +++ b/tools/checks/structure.py @@ -12,7 +12,7 @@ through absolute imports, relative imports, and imports inside functions or `if`/`try` blocks. Imports under `if TYPE_CHECKING:` are typing-only: a cycle made only of them is not reported. -3. FACADE_IMPORTED (a failure): an internal module importing its package's façade (`store.lake`, +3. FACADE_IMPORTED (a failure): an internal module importing its package's façade (`store.api`, `server.commands`). A cycle or a façade import that exists before its split (#246, PR 2 and 3) is listed in `file-size-exceptions.toml` (`[cycle]`, `[facade]`) with the issue that removes it; an entry that no longer applies fails, so the lists only shrink. @@ -35,7 +35,8 @@ LIMIT = 800 EXCEPTIONS = Path(__file__).with_name("file-size-exceptions.toml") -FACADES = {"store": "lake", "server": "commands"} +# Each package's public face; the store's is its contract implementation (#486). +FACADES = {"store": "api", "server": "commands"} @dataclass(frozen=True, slots=True) From 3b7e9274ca5db6444e2e9717e3d6753f86a22d09 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:23:31 -0400 Subject: [PATCH 21/21] feat(store): the extension and engine-version codes are declared, proven in a process of their own `open_lake` and `open_existing_lake` declare STORE_EXTENSION_UNAVAILABLE, proven with an empty extension directory and a proxy nothing answers on; `connect_clickhouse` declares ENGINE_VERSION_UNTESTED, proven against a ClickHouse 26.9 service the kit's CI jobs start. Each case declares its environment and runs in its own process (the foundations' kit_process), so no case patches the store; their checks live beside the kit, where that process finds them. Part of #486 (step 2, store). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .github/workflows/interfaces.yml | 13 ++ .github/workflows/python.yml | 13 ++ packages/contracts/interface-record.json | 204 +++++++++++++++++- .../src/orvex/contracts/store/interface.py | 3 + .../src/orvex/contracts/store/kit.py | 64 +++++- .../src/orvex/contracts/store/kit_codes.py | 38 +++- .../contracts/tests/test_store_kit_codes.py | 6 +- tools/ci/checks.toml | 4 +- 8 files changed, 328 insertions(+), 17 deletions(-) diff --git a/.github/workflows/interfaces.yml b/.github/workflows/interfaces.yml index 8adf18c2..1b6ef78d 100644 --- a/.github/workflows/interfaces.yml +++ b/.github/workflows/interfaces.yml @@ -85,6 +85,19 @@ jobs: options: >- --health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'" --health-interval 2s --health-timeout 5s --health-retries 60 + # A ClickHouse of a version the store does not test, for the store kit's + # ENGINE_VERSION_UNTESTED case (#486); a disposable server, not a secret. + clickhouse-untested: + # clickhouse/clickhouse-server:26.9.25.15 (between the tested 26.8 and 26.10) + image: clickhouse/clickhouse-server@sha256:19bfacb135479a892551a8155cfbbc8211aedf2aa888cd91950b9ee7e76cc7e3 + env: + CLICKHOUSE_USER: ci + CLICKHOUSE_PASSWORD: ci-disposable-database + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + ports: ["8124:8123"] + options: >- + --health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'" + --health-interval 2s --health-timeout 5s --health-retries 60 env: STORE_TEST_POSTGRES: host=localhost port=5432 dbname=postgres user=postgres STORE_TEST_POSTGRES_PASSWORD: ci-disposable-database diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 89f2368e..f7472e8a 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -95,6 +95,19 @@ jobs: options: >- --health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'" --health-interval 2s --health-timeout 5s --health-retries 60 + # A ClickHouse of a version the store does not test, for the store kit's + # ENGINE_VERSION_UNTESTED case (#486); a disposable server, not a secret. + clickhouse-untested: + # clickhouse/clickhouse-server:26.9.25.15 (between the tested 26.8 and 26.10) + image: clickhouse/clickhouse-server@sha256:19bfacb135479a892551a8155cfbbc8211aedf2aa888cd91950b9ee7e76cc7e3 + env: + CLICKHOUSE_USER: ci + CLICKHOUSE_PASSWORD: ci-disposable-database + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + ports: ["8124:8123"] + options: >- + --health-cmd "clickhouse-client --user ci --password ci-disposable-database --query 'SELECT 1'" + --health-interval 2s --health-timeout 5s --health-retries 60 env: STORE_TEST_POSTGRES: host=localhost port=5432 dbname=postgres user=postgres STORE_TEST_POSTGRES_PASSWORD: ci-disposable-database diff --git a/packages/contracts/interface-record.json b/packages/contracts/interface-record.json index d17e5366..3e0f5dc1 100644 --- a/packages/contracts/interface-record.json +++ b/packages/contracts/interface-record.json @@ -41,925 +41,1124 @@ "kit": { "cases": { "begin.idempotent": { + "environment": null, "expectation": "one key begins one transaction", "idempotency": true, "operation": "begin", "since": "1.0" }, "begin.raises.store-busy": { + "environment": null, "expectation": "begin refuses with STORE_BUSY", "idempotency": false, "operation": "begin", "since": "1.0" }, "begin.raises.store-parameter-invalid": { + "environment": null, "expectation": "begin refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "begin", "since": "1.0" }, "begin.raises.store-statement-invalid": { + "environment": null, "expectation": "begin refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "begin", "since": "1.0" }, "begin.raises.store-timeout": { + "environment": null, "expectation": "begin refuses with STORE_TIMEOUT", "idempotency": false, "operation": "begin", "since": "1.0" }, "cancel-read.idempotent": { + "environment": null, "expectation": "one key cancels once", "idempotency": true, "operation": "cancel_read", "since": "1.0" }, "cancel-read.raises.store-parameter-invalid": { + "environment": null, "expectation": "cancel_read refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "cancel_read", "since": "1.0" }, "cancel-transaction.idempotent": { + "environment": null, "expectation": "one key cancels once", "idempotency": true, "operation": "cancel_transaction", "since": "1.0" }, "cancel-transaction.raises.store-parameter-invalid": { + "environment": null, "expectation": "cancel_transaction refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "cancel_transaction", "since": "1.0" }, "cancel-transaction.rolls-back": { + "environment": null, "expectation": "a cancelled transaction never commits", "idempotency": false, "operation": "cancel_transaction", "since": "1.0" }, "changes.raises.store-busy": { + "environment": null, "expectation": "changes refuses with STORE_BUSY", "idempotency": false, "operation": "changes", "since": "1.0" }, "changes.raises.store-parameter-invalid": { + "environment": null, "expectation": "changes refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "changes", "since": "1.0" }, "changes.raises.store-snapshot-expired": { + "environment": null, "expectation": "changes refuses with STORE_SNAPSHOT_EXPIRED", "idempotency": false, "operation": "changes", "since": "1.0" }, "changes.raises.store-snapshot-not-found": { + "environment": null, "expectation": "changes refuses with STORE_SNAPSHOT_NOT_FOUND", "idempotency": false, "operation": "changes", "since": "1.0" }, "changes.raises.store-statement-invalid": { + "environment": null, "expectation": "changes refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "changes", "since": "1.0" }, "changes.removed": { + "environment": null, "expectation": "removed rows are counted; `since` after `snapshot` refused", "idempotency": false, "operation": "changes", "since": "1.0" }, "check-apart.protected": { + "environment": null, "expectation": "a lake inside a protected location is refused", "idempotency": false, "operation": "check_apart", "since": "1.0" }, "check-apart.raises.store-source-invalid": { + "environment": null, "expectation": "check_apart refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "check_apart", "since": "1.0" }, "clickhouse-catalog.raises.store-parameter-invalid": { + "environment": null, "expectation": "clickhouse_catalog refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_catalog", "since": "1.0" }, "clickhouse-columns.raises.source-not-found": { + "environment": null, "expectation": "clickhouse_columns refuses with SOURCE_NOT_FOUND", "idempotency": false, "operation": "clickhouse_columns", "since": "1.0" }, "clickhouse-columns.raises.store-parameter-invalid": { + "environment": null, "expectation": "clickhouse_columns refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_columns", "since": "1.0" }, "clickhouse-columns.raises.unsupported-source-type": { + "environment": null, "expectation": "clickhouse_columns refuses with UNSUPPORTED_SOURCE_TYPE", "idempotency": false, "operation": "clickhouse_columns", "since": "1.0" }, "clickhouse-estimate.raises.store-parameter-invalid": { + "environment": null, "expectation": "clickhouse_estimate refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_estimate", "since": "1.0" }, "clickhouse-estimate.raises.store-statement-invalid": { + "environment": null, "expectation": "clickhouse_estimate refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "clickhouse_estimate", "since": "1.0" }, "clickhouse-now.raises.store-parameter-invalid": { + "environment": null, "expectation": "clickhouse_now refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "clickhouse_now", "since": "1.0" }, "close-clickhouse.harmless": { + "environment": null, "expectation": "closing what is not held changes nothing", "idempotency": false, "operation": "close_clickhouse", "since": "1.0" }, "close-existing-lake.idempotent": { + "environment": null, "expectation": "one key detaches once", "idempotency": true, "operation": "close_existing_lake", "since": "1.0" }, "close-existing-lake.raises.store-parameter-invalid": { + "environment": null, "expectation": "close_existing_lake refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "close_existing_lake", "since": "1.0" }, "close-lake.idempotent": { + "environment": null, "expectation": "one key closes once", "idempotency": true, "operation": "close_lake", "since": "1.0" }, "close-lake.raises.store-parameter-invalid": { + "environment": null, "expectation": "close_lake refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "close_lake", "since": "1.0" }, "close-read.idempotent": { + "environment": null, "expectation": "one key lets a read go once", "idempotency": true, "operation": "close_read", "since": "1.0" }, "close-read.raises.store-parameter-invalid": { + "environment": null, "expectation": "close_read refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "close_read", "since": "1.0" }, "commit.idempotent": { + "environment": null, "expectation": "one key commits once and answers the same", "idempotency": true, "operation": "commit", "since": "1.0" }, "commit.raises.store-cancelled": { + "environment": null, "expectation": "commit refuses with STORE_CANCELLED", "idempotency": false, "operation": "commit", "since": "1.0" }, "commit.raises.store-parameter-invalid": { + "environment": null, "expectation": "commit refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "commit", "since": "1.0" }, "commit.raises.store-query-failed": { + "environment": null, "expectation": "commit refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "commit", "since": "1.0" }, "commit.raises.store-timeout": { + "environment": null, "expectation": "commit refuses with STORE_TIMEOUT", "idempotency": false, "operation": "commit", "since": "1.0" }, "commit.raises.store-transaction-conflict": { + "environment": null, "expectation": "commit refuses with STORE_TRANSACTION_CONFLICT", "idempotency": false, "operation": "commit", "since": "1.0" }, "connect-clickhouse.raises.engine-setting-required": { + "environment": null, "expectation": "connect_clickhouse refuses with ENGINE_SETTING_REQUIRED", "idempotency": false, "operation": "connect_clickhouse", "since": "1.0" }, + "connect-clickhouse.raises.engine-version-untested": { + "environment": { + "extensions": null, + "variables": { + "STORE_KIT_UNTESTED_PASSWORD": "ci-disposable-database" + } + }, + "expectation": "connect_clickhouse refuses with ENGINE_VERSION_UNTESTED, in the environment the case declares", + "idempotency": false, + "operation": "connect_clickhouse", + "since": "1.0" + }, "connect-clickhouse.raises.source-not-read-only": { + "environment": null, "expectation": "connect_clickhouse refuses with SOURCE_NOT_READ_ONLY", "idempotency": false, "operation": "connect_clickhouse", "since": "1.0" }, "connect-clickhouse.raises.store-connection-failed": { + "environment": null, "expectation": "connect_clickhouse refuses with STORE_CONNECTION_FAILED", "idempotency": false, "operation": "connect_clickhouse", "since": "1.0" }, "connect-clickhouse.raises.store-secret-missing": { + "environment": null, "expectation": "connect_clickhouse refuses with STORE_SECRET_MISSING", "idempotency": false, "operation": "connect_clickhouse", "since": "1.0" }, "connect-clickhouse.raises.store-tls-required": { + "environment": null, "expectation": "connect_clickhouse refuses with STORE_TLS_REQUIRED", "idempotency": false, "operation": "connect_clickhouse", "since": "1.0" }, "current-snapshot.after-commit": { + "environment": null, "expectation": "the snapshot a commit made is current", "idempotency": false, "operation": "current_snapshot", "since": "1.0" }, "current-snapshot.raises.store-parameter-invalid": { + "environment": null, "expectation": "current_snapshot refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "current_snapshot", "since": "1.0" }, "describe-source.landed-types": { + "environment": null, "expectation": "a source's columns as they land; an unknown source refused", "idempotency": false, "operation": "describe_source", "since": "1.0" }, "describe-source.raises.store-parameter-invalid": { + "environment": null, "expectation": "describe_source refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "describe_source", "since": "1.0" }, "describe-source.raises.store-query-failed": { + "environment": null, "expectation": "describe_source refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "describe_source", "since": "1.0" }, "describe-source.raises.store-source-invalid": { + "environment": null, "expectation": "describe_source refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "describe_source", "since": "1.0" }, "describe-source.raises.store-statement-invalid": { + "environment": null, "expectation": "describe_source refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "describe_source", "since": "1.0" }, "describe.binder-only": { + "environment": null, "expectation": "a SELECT's columns from the binder; anything else refused", "idempotency": false, "operation": "describe", "since": "1.0" }, "describe.raises.store-parameter-invalid": { + "environment": null, "expectation": "describe refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "describe", "since": "1.0" }, "describe.raises.store-query-failed": { + "environment": null, "expectation": "describe refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "describe", "since": "1.0" }, "describe.raises.store-statement-invalid": { + "environment": null, "expectation": "describe refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "describe", "since": "1.0" }, "execute.idempotent": { + "environment": null, "expectation": "one key inserts once", "idempotency": true, "operation": "execute", "since": "1.0" }, "execute.raises.store-cancelled": { + "environment": null, "expectation": "execute refuses with STORE_CANCELLED", "idempotency": false, "operation": "execute", "since": "1.0" }, "execute.raises.store-parameter-invalid": { + "environment": null, "expectation": "execute refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "execute", "since": "1.0" }, "execute.raises.store-query-failed": { + "environment": null, "expectation": "execute refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "execute", "since": "1.0" }, "execute.raises.store-statement-invalid": { + "environment": null, "expectation": "execute refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "execute", "since": "1.0" }, "execute.raises.store-timeout": { + "environment": null, "expectation": "execute refuses with STORE_TIMEOUT", "idempotency": false, "operation": "execute", "since": "1.0" }, "existing-committed-at.raises.store-parameter-invalid": { + "environment": null, "expectation": "existing_committed_at refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_committed_at", "since": "1.0" }, "existing-latest.raises.store-parameter-invalid": { + "environment": null, "expectation": "existing_latest refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_latest", "since": "1.0" }, "existing-latest.reads": { + "environment": null, "expectation": "latest, verify, committed-at, columns, views and a read, exact", "idempotency": false, "operation": "existing_latest", "since": "1.0" }, "existing-table-columns.raises.store-parameter-invalid": { + "environment": null, "expectation": "existing_table_columns refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_table_columns", "since": "1.0" }, "existing-table-columns.raises.store-source-invalid": { + "environment": null, "expectation": "existing_table_columns refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "existing_table_columns", "since": "1.0" }, "existing-verify.raises.store-parameter-invalid": { + "environment": null, "expectation": "existing_verify refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_verify", "since": "1.0" }, "existing-verify.raises.store-source-invalid": { + "environment": null, "expectation": "existing_verify refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "existing_verify", "since": "1.0" }, "existing-views.raises.store-parameter-invalid": { + "environment": null, "expectation": "existing_views refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "existing_views", "since": "1.0" }, "held.lists-open": { + "environment": null, "expectation": "what is held is listed; a closed lake is let go and refused", "idempotency": false, "operation": "held", "since": "1.0" }, "is-object-storage.schemes": { + "environment": null, "expectation": "s3://, gs:// and az:// only, case as written", "idempotency": false, "operation": "is_object_storage", "since": "1.0" }, "json-frame.columns": { + "environment": null, "expectation": "the columns with their DuckDB types and the closing tail; no type is refused", "idempotency": false, "operation": "json_frame", "since": "1.0" }, "json-frame.raises.store-type-unsupported": { + "environment": null, "expectation": "json_frame refuses with STORE_TYPE_UNSUPPORTED", "idempotency": false, "operation": "json_frame", "since": "1.0" }, "json-rows.exact": { + "environment": null, "expectation": "exact text, NULL as null, rows separated", "idempotency": false, "operation": "json_rows", "since": "1.0" }, "json-rows.raises.store-type-unsupported": { + "environment": null, "expectation": "json_rows refuses with STORE_TYPE_UNSUPPORTED", "idempotency": false, "operation": "json_rows", "since": "1.0" }, "land.idempotent": { + "environment": null, "expectation": "one key lands once", "idempotency": true, "operation": "land", "since": "1.0" }, "land.raises.store-cancelled": { + "environment": null, "expectation": "land refuses with STORE_CANCELLED", "idempotency": false, "operation": "land", "since": "1.0" }, "land.raises.store-parameter-invalid": { + "environment": null, "expectation": "land refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "land", "since": "1.0" }, "land.raises.store-query-failed": { + "environment": null, "expectation": "land refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "land", "since": "1.0" }, "land.raises.store-source-invalid": { + "environment": null, "expectation": "land refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "land", "since": "1.0" }, "land.raises.store-timeout": { + "environment": null, "expectation": "land refuses with STORE_TIMEOUT", "idempotency": false, "operation": "land", "since": "1.0" }, "load-arrow.exact": { + "environment": null, "expectation": "a DECIMAL and a NULL are loaded exactly", "idempotency": false, "operation": "load_arrow", "since": "1.0" }, "load-arrow.idempotent": { + "environment": null, "expectation": "one key loads once", "idempotency": true, "operation": "load_arrow", "since": "1.0" }, "load-arrow.raises.store-cancelled": { + "environment": null, "expectation": "load_arrow refuses with STORE_CANCELLED", "idempotency": false, "operation": "load_arrow", "since": "1.0" }, "load-arrow.raises.store-parameter-invalid": { + "environment": null, "expectation": "load_arrow refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "load_arrow", "since": "1.0" }, "load-arrow.raises.store-query-failed": { + "environment": null, "expectation": "load_arrow refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "load_arrow", "since": "1.0" }, "load-arrow.raises.store-timeout": { + "environment": null, "expectation": "load_arrow refuses with STORE_TIMEOUT", "idempotency": false, "operation": "load_arrow", "since": "1.0" }, "location-components.parts": { + "environment": null, "expectation": "object storage and local paths by parts; anything else refused", "idempotency": false, "operation": "location_components", "since": "1.0" }, "location-components.raises.store-source-invalid": { + "environment": null, "expectation": "location_components refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "location_components", "since": "1.0" }, "locations-overlap.prefix": { + "environment": null, "expectation": "inside overlaps, a sibling with the same prefix does not", "idempotency": false, "operation": "locations_overlap", "since": "1.0" }, "locations-overlap.raises.store-source-invalid": { + "environment": null, "expectation": "locations_overlap refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "locations_overlap", "since": "1.0" }, "maintain.idempotent": { + "environment": null, "expectation": "one key maintains once and answers the same", "idempotency": true, "operation": "maintain", "since": "1.0" }, "maintain.raises.store-busy": { + "environment": null, "expectation": "maintain refuses with STORE_BUSY", "idempotency": false, "operation": "maintain", "since": "1.0" }, "maintain.raises.store-parameter-invalid": { + "environment": null, "expectation": "maintain refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "maintain", "since": "1.0" }, "open-existing-lake.idempotent": { + "environment": null, "expectation": "one key attaches once", "idempotency": true, "operation": "open_existing_lake", "since": "1.0" }, "open-existing-lake.raises.engine-version-untested": { + "environment": null, "expectation": "open_existing_lake refuses with ENGINE_VERSION_UNTESTED", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, "open-existing-lake.raises.store-connection-failed": { + "environment": null, "expectation": "open_existing_lake refuses with STORE_CONNECTION_FAILED", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, + "open-existing-lake.raises.store-extension-unavailable": { + "environment": { + "extensions": { + "source": null, + "variable": "HOME", + "without": [] + }, + "variables": { + "HTTPS_PROXY": "http://127.0.0.1:1", + "HTTP_PROXY": "http://127.0.0.1:1" + } + }, + "expectation": "open_existing_lake refuses with STORE_EXTENSION_UNAVAILABLE, in the environment the case declares", + "idempotency": false, + "operation": "open_existing_lake", + "since": "1.0" + }, "open-existing-lake.raises.store-limits-required": { + "environment": null, "expectation": "open_existing_lake refuses with STORE_LIMITS_REQUIRED", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, "open-existing-lake.raises.store-location-invalid": { + "environment": null, "expectation": "open_existing_lake refuses with STORE_LOCATION_INVALID", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, "open-existing-lake.raises.store-secret-missing": { + "environment": null, "expectation": "open_existing_lake refuses with STORE_SECRET_MISSING", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, "open-existing-lake.raises.store-source-invalid": { + "environment": null, "expectation": "open_existing_lake refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, "open-existing-lake.raises.store-tls-required": { + "environment": null, "expectation": "open_existing_lake refuses with STORE_TLS_REQUIRED", "idempotency": false, "operation": "open_existing_lake", "since": "1.0" }, "open-lake.idempotent": { + "environment": null, "expectation": "one key opens one lake", "idempotency": true, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.source-not-found": { + "environment": null, "expectation": "open_lake refuses with SOURCE_NOT_FOUND", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.store-catalog-unavailable": { + "environment": null, "expectation": "open_lake refuses with STORE_CATALOG_UNAVAILABLE", "idempotency": false, "operation": "open_lake", "since": "1.0" }, + "open-lake.raises.store-extension-unavailable": { + "environment": { + "extensions": { + "source": null, + "variable": "HOME", + "without": [] + }, + "variables": { + "HTTPS_PROXY": "http://127.0.0.1:1", + "HTTP_PROXY": "http://127.0.0.1:1" + } + }, + "expectation": "open_lake refuses with STORE_EXTENSION_UNAVAILABLE, in the environment the case declares", + "idempotency": false, + "operation": "open_lake", + "since": "1.0" + }, "open-lake.raises.store-limits-required": { + "environment": null, "expectation": "open_lake refuses with STORE_LIMITS_REQUIRED", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.store-location-invalid": { + "environment": null, "expectation": "open_lake refuses with STORE_LOCATION_INVALID", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.store-secret-inline": { + "environment": null, "expectation": "open_lake refuses with STORE_SECRET_INLINE", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.store-secret-missing": { + "environment": null, "expectation": "open_lake refuses with STORE_SECRET_MISSING", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.store-source-invalid": { + "environment": null, "expectation": "open_lake refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "open-lake.raises.store-tls-required": { + "environment": null, "expectation": "open_lake refuses with STORE_TLS_REQUIRED", "idempotency": false, "operation": "open_lake", "since": "1.0" }, "read-batch.ends": { + "environment": null, "expectation": "every row once, then None; a forged read is refused", "idempotency": false, "operation": "read_batch", "since": "1.0" }, "read-batch.raises.store-cancelled": { + "environment": null, "expectation": "read_batch refuses with STORE_CANCELLED", "idempotency": false, "operation": "read_batch", "since": "1.0" }, "read-batch.raises.store-parameter-invalid": { + "environment": null, "expectation": "read_batch refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "read_batch", "since": "1.0" }, "read-batch.raises.store-query-failed": { + "environment": null, "expectation": "read_batch refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "read_batch", "since": "1.0" }, "read-batch.raises.store-timeout": { + "environment": null, "expectation": "read_batch refuses with STORE_TIMEOUT", "idempotency": false, "operation": "read_batch", "since": "1.0" }, "rollback.discards": { + "environment": null, "expectation": "a rolled-back transaction writes nothing and is let go", "idempotency": false, "operation": "rollback", "since": "1.0" }, "rollback.idempotent": { + "environment": null, "expectation": "one key rolls back once", "idempotency": true, "operation": "rollback", "since": "1.0" }, "rollback.raises.store-parameter-invalid": { + "environment": null, "expectation": "rollback refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "rollback", "since": "1.0" }, "snapshot.kept-or-none": { + "environment": null, "expectation": "a kept snapshot is found; an unknown or negative id is None", "idempotency": false, "operation": "snapshot", "since": "1.0" }, "snapshot.raises.store-parameter-invalid": { + "environment": null, "expectation": "snapshot refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "snapshot", "since": "1.0" }, "start-clickhouse-read.raises.store-parameter-invalid": { + "environment": null, "expectation": "start_clickhouse_read refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "start_clickhouse_read", "since": "1.0" }, "start-clickhouse-read.raises.store-query-failed": { + "environment": null, "expectation": "start_clickhouse_read refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "start_clickhouse_read", "since": "1.0" }, "start-clickhouse-read.raises.store-statement-invalid": { + "environment": null, "expectation": "start_clickhouse_read refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "start_clickhouse_read", "since": "1.0" }, "start-existing-read.raises.store-parameter-invalid": { + "environment": null, "expectation": "start_existing_read refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "start_existing_read", "since": "1.0" }, "start-existing-read.raises.store-query-failed": { + "environment": null, "expectation": "start_existing_read refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "start_existing_read", "since": "1.0" }, "start-existing-read.raises.store-source-invalid": { + "environment": null, "expectation": "start_existing_read refuses with STORE_SOURCE_INVALID", "idempotency": false, "operation": "start_existing_read", "since": "1.0" }, "start-existing-read.raises.store-statement-invalid": { + "environment": null, "expectation": "start_existing_read refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "start_existing_read", "since": "1.0" }, "start-read.exact": { + "environment": null, "expectation": "DECIMAL, HUGEINT, TIMESTAMPTZ and NULL are read exactly", "idempotency": false, "operation": "start_read", "since": "1.0" }, "start-read.parameters": { + "environment": null, "expectation": "a parameter is bound by name with its type", "idempotency": false, "operation": "start_read", "since": "1.0" }, "start-read.raises.store-busy": { + "environment": null, "expectation": "start_read refuses with STORE_BUSY", "idempotency": false, "operation": "start_read", "since": "1.0" }, "start-read.raises.store-parameter-invalid": { + "environment": null, "expectation": "start_read refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "start_read", "since": "1.0" }, "start-read.raises.store-snapshot-expired": { + "environment": null, "expectation": "start_read refuses with STORE_SNAPSHOT_EXPIRED", "idempotency": false, "operation": "start_read", "since": "1.0" }, "start-read.raises.store-snapshot-not-found": { + "environment": null, "expectation": "start_read refuses with STORE_SNAPSHOT_NOT_FOUND", "idempotency": false, "operation": "start_read", "since": "1.0" }, "start-read.raises.store-statement-invalid": { + "environment": null, "expectation": "start_read refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "start_read", "since": "1.0" }, "statistics.counts": { + "environment": null, "expectation": "every row is counted; an unknown table refused", "idempotency": false, "operation": "statistics", "since": "1.0" }, "statistics.raises.store-busy": { + "environment": null, "expectation": "statistics refuses with STORE_BUSY", "idempotency": false, "operation": "statistics", "since": "1.0" }, "statistics.raises.store-parameter-invalid": { + "environment": null, "expectation": "statistics refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "statistics", "since": "1.0" }, "statistics.raises.store-snapshot-expired": { + "environment": null, "expectation": "statistics refuses with STORE_SNAPSHOT_EXPIRED", "idempotency": false, "operation": "statistics", "since": "1.0" }, "statistics.raises.store-snapshot-not-found": { + "environment": null, "expectation": "statistics refuses with STORE_SNAPSHOT_NOT_FOUND", "idempotency": false, "operation": "statistics", "since": "1.0" }, "statistics.raises.store-statement-invalid": { + "environment": null, "expectation": "statistics refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "statistics", "since": "1.0" }, "table-columns.as-of": { + "environment": null, "expectation": "the columns as of the snapshot asked", "idempotency": false, "operation": "table_columns", "since": "1.0" }, "table-columns.raises.store-busy": { + "environment": null, "expectation": "table_columns refuses with STORE_BUSY", "idempotency": false, "operation": "table_columns", "since": "1.0" }, "table-columns.raises.store-parameter-invalid": { + "environment": null, "expectation": "table_columns refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "table_columns", "since": "1.0" }, "table-columns.raises.store-snapshot-expired": { + "environment": null, "expectation": "table_columns refuses with STORE_SNAPSHOT_EXPIRED", "idempotency": false, "operation": "table_columns", "since": "1.0" }, "table-columns.raises.store-snapshot-not-found": { + "environment": null, "expectation": "table_columns refuses with STORE_SNAPSHOT_NOT_FOUND", "idempotency": false, "operation": "table_columns", "since": "1.0" }, "tables.lower-cased": { + "environment": null, "expectation": "every table, lower-cased, in order", "idempotency": false, "operation": "tables", "since": "1.0" }, "tables.raises.store-parameter-invalid": { + "environment": null, "expectation": "tables refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "tables", "since": "1.0" }, "value.first-cell": { + "environment": null, "expectation": "the first cell as text; None when there is no row", "idempotency": false, "operation": "value", "since": "1.0" }, "value.raises.store-cancelled": { + "environment": null, "expectation": "value refuses with STORE_CANCELLED", "idempotency": false, "operation": "value", "since": "1.0" }, "value.raises.store-parameter-invalid": { + "environment": null, "expectation": "value refuses with STORE_PARAMETER_INVALID", "idempotency": false, "operation": "value", "since": "1.0" }, "value.raises.store-query-failed": { + "environment": null, "expectation": "value refuses with STORE_QUERY_FAILED", "idempotency": false, "operation": "value", "since": "1.0" }, "value.raises.store-statement-invalid": { + "environment": null, "expectation": "value refuses with STORE_STATEMENT_INVALID", "idempotency": false, "operation": "value", "since": "1.0" }, "value.raises.store-timeout": { + "environment": null, "expectation": "value refuses with STORE_TIMEOUT", "idempotency": false, "operation": "value", "since": "1.0" } }, - "recorded": 153 + "recorded": 156 }, "name": "store", "operations": { @@ -1350,6 +1549,7 @@ "changes_state": false, "errors": [ "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", "SOURCE_NOT_READ_ONLY", "STORE_CONNECTION_FAILED", "STORE_SECRET_MISSING", @@ -1975,6 +2175,7 @@ "errors": [ "ENGINE_VERSION_UNTESTED", "STORE_CONNECTION_FAILED", + "STORE_EXTENSION_UNAVAILABLE", "STORE_LIMITS_REQUIRED", "STORE_LOCATION_INVALID", "STORE_SECRET_MISSING", @@ -2010,6 +2211,7 @@ "errors": [ "SOURCE_NOT_FOUND", "STORE_CATALOG_UNAVAILABLE", + "STORE_EXTENSION_UNAVAILABLE", "STORE_LIMITS_REQUIRED", "STORE_LOCATION_INVALID", "STORE_SECRET_INLINE", diff --git a/packages/contracts/src/orvex/contracts/store/interface.py b/packages/contracts/src/orvex/contracts/store/interface.py index dfe82065..e1cc85d4 100644 --- a/packages/contracts/src/orvex/contracts/store/interface.py +++ b/packages/contracts/src/orvex/contracts/store/interface.py @@ -330,6 +330,7 @@ def check_apart( ERRORS_OF: dict[str, tuple[str, ...]] = { "open_lake": ( "STORE_LIMITS_REQUIRED", + "STORE_EXTENSION_UNAVAILABLE", "STORE_LOCATION_INVALID", "STORE_SECRET_MISSING", "STORE_SECRET_INLINE", @@ -375,6 +376,7 @@ def check_apart( "maintain": (HANDLE, *SLOT), "open_existing_lake": ( "STORE_LIMITS_REQUIRED", + "STORE_EXTENSION_UNAVAILABLE", "STORE_LOCATION_INVALID", "STORE_SECRET_MISSING", "STORE_TLS_REQUIRED", @@ -395,6 +397,7 @@ def check_apart( "STORE_CONNECTION_FAILED", "SOURCE_NOT_READ_ONLY", "ENGINE_SETTING_REQUIRED", + "ENGINE_VERSION_UNTESTED", ), "close_clickhouse": (), "clickhouse_columns": (HANDLE, "SOURCE_NOT_FOUND", "UNSUPPORTED_SOURCE_TYPE"), diff --git a/packages/contracts/src/orvex/contracts/store/kit.py b/packages/contracts/src/orvex/contracts/store/kit.py index 72dd64f5..f6d06801 100644 --- a/packages/contracts/src/orvex/contracts/store/kit.py +++ b/packages/contracts/src/orvex/contracts/store/kit.py @@ -17,7 +17,7 @@ import pyarrow as pa from orvex.contracts.envelope import CallEnvelope -from orvex.contracts.kit import Idempotent, Kit, KitCase +from orvex.contracts.kit import CaseEnvironment, ExtensionDirectory, Idempotent, Kit, KitCase from orvex.contracts.kit_client import KitClient from orvex.contracts.store.interface import STORE from orvex.contracts.store.kit_codes import code_cases @@ -29,6 +29,7 @@ commit, drain, existing_lake, + folder, local_location, open_lake, read_request, @@ -40,13 +41,16 @@ ) from orvex.contracts.store.v1 import ( ChangesRequest, + ExistingLakeAttachment, Parameter, + SqliteLakeLocation, StatisticsRequest, TablesRequest, ) from orvex.contracts.store.values import ( Budget, ClickHouseHandle, + ClickHouseTarget, ExistingLakeHandle, LakeHandle, ReadHandle, @@ -717,5 +721,59 @@ def _case( ), ) -RECORDED = 153 -STORE_KIT = Kit(contract=STORE, recorded=RECORDED, cases=(*CASES, *code_cases(STORE))) +# --- the code cases that run in a process of their own (found here by that process) --------------- + +# No extension at hand and none to download: DuckDB's extension directory is under an empty +# home, and every download goes to a proxy nothing answers on. +DEAD_PROXY = "http://127.0.0.1:1" +WITHHELD = CaseEnvironment( + variables=(("HTTP_PROXY", DEAD_PROXY), ("HTTPS_PROXY", DEAD_PROXY)), + extensions=ExtensionDirectory("HOME"), +) +# A ClickHouse server of a version the store does not test, the CI job's service on port 8124 +# (a disposable server, its password not a secret). +UNTESTED_PASSWORD = "STORE_KIT_UNTESTED_PASSWORD" +UNTESTED = CaseEnvironment(variables=((UNTESTED_PASSWORD, "ci-disposable-database"),)) + + +def extension_withheld_lake(client: KitClient) -> None: + refused("STORE_EXTENSION_UNAVAILABLE", lambda: client.open_lake(local_location(), LIMITS, ())) + + +def extension_withheld_existing(client: KitClient) -> None: + root = folder() + location = SqliteLakeLocation( + kind="lake_location", + catalog="sqlite", + path=str(root / "catalog.sqlite"), + data_path=(root / "data").as_posix() + "/", + ) + attachment = ExistingLakeAttachment( + kind="existing_lake_attachment", + name="owner", + location=location, + roles_in_use=[], + protected_files=[], + credentials_in_use=[], + protected_locations=[], + ) + refused( + "STORE_EXTENSION_UNAVAILABLE", + lambda: client.open_existing_lake(attachment, EXISTING_LIMITS), + ) + + +def untested_clickhouse(client: KitClient) -> None: + target = ClickHouseTarget("127.0.0.1", 8124, "default", "ci", UNTESTED_PASSWORD, tls=False) + refused("ENGINE_VERSION_UNTESTED", lambda: client.connect_clickhouse(target, BUDGET)) + + +ISOLATED: dict[tuple[str, str], tuple[Callable[[KitClient], None], CaseEnvironment]] = { + ("open_lake", "STORE_EXTENSION_UNAVAILABLE"): (extension_withheld_lake, WITHHELD), + ("open_existing_lake", "STORE_EXTENSION_UNAVAILABLE"): (extension_withheld_existing, WITHHELD), + ("connect_clickhouse", "ENGINE_VERSION_UNTESTED"): (untested_clickhouse, UNTESTED), +} + + +RECORDED = 156 +STORE_KIT = Kit(contract=STORE, recorded=RECORDED, cases=(*CASES, *code_cases(STORE, ISOLATED))) diff --git a/packages/contracts/src/orvex/contracts/store/kit_codes.py b/packages/contracts/src/orvex/contracts/store/kit_codes.py index 69a57978..4efcb907 100644 --- a/packages/contracts/src/orvex/contracts/store/kit_codes.py +++ b/packages/contracts/src/orvex/contracts/store/kit_codes.py @@ -16,7 +16,7 @@ import pyarrow as pa from orvex.contracts.interface import InterfaceContract from orvex.contracts.interface_errors import InterfaceContractError -from orvex.contracts.kit import KitCase +from orvex.contracts.kit import CaseEnvironment, KitCase from orvex.contracts.kit_client import KitClient from orvex.contracts.store.kit_services import CLICKHOUSE_PASSWORD, SAFE, clickhouse from orvex.contracts.store.kit_support import ( @@ -731,19 +731,38 @@ def provoke(client: KitClient) -> Callable[[], object]: } +def _identifier(operation: str, code: str) -> str: + return f"{operation.replace('_', '-')}.raises.{code.lower().replace('_', '-')}" + + def _case(operation: str, code: str, provoke: Provoke) -> KitCase: def check(client: KitClient) -> None: refused(code, provoke(client)) - identifier = f"{operation.replace('_', '-')}.raises.{code.lower().replace('_', '-')}" - return KitCase(identifier, operation, f"{operation} refuses with {code}", check) + return KitCase( + _identifier(operation, code), operation, f"{operation} refuses with {code}", check + ) + + +type Isolated = dict[tuple[str, str], tuple[Callable[[KitClient], None], CaseEnvironment]] + + +def _isolated(operation: str, code: str, isolated: Isolated) -> KitCase: + check, environment = isolated[operation, code] + expectation = f"{operation} refuses with {code}, in the environment the case declares" + return KitCase( + _identifier(operation, code), operation, expectation, check, environment=environment + ) -def code_cases(contract: InterfaceContract) -> tuple[KitCase, ...]: - """One case per declared (operation, code); KIT_CODE_UNPROVEN when the two differ.""" +def code_cases(contract: InterfaceContract, isolated: Isolated) -> tuple[KitCase, ...]: + """One case per declared (operation, code), those of `isolated` each in the environment it + declares (their checks live beside the kit, where a case's own process finds it); + KIT_CODE_UNPROVEN when the declared and the proven codes differ.""" declared = {(o.name, code) for o in contract.operations for code in o.errors} - unproven = sorted(declared - set(PROOFS)) - undeclared = sorted(set(PROOFS) - declared) + proven = set(PROOFS) | set(isolated) + unproven = sorted(declared - proven) + undeclared = sorted(proven - declared) if unproven or undeclared: raise InterfaceContractError( "KIT_CODE_UNPROVEN", @@ -752,5 +771,8 @@ def code_cases(contract: InterfaceContract) -> tuple[KitCase, ...]: {"contract": contract.name}, ) return tuple( - _case(operation, code, PROOFS[operation, code]) for operation, code in sorted(PROOFS) + _isolated(operation, code, isolated) + if (operation, code) in isolated + else _case(operation, code, PROOFS[operation, code]) + for operation, code in sorted(proven) ) diff --git a/packages/contracts/tests/test_store_kit_codes.py b/packages/contracts/tests/test_store_kit_codes.py index b046e0ef..9d163422 100644 --- a/packages/contracts/tests/test_store_kit_codes.py +++ b/packages/contracts/tests/test_store_kit_codes.py @@ -8,7 +8,7 @@ import pytest from orvex.contracts.interface_errors import InterfaceContractError from orvex.contracts.store.interface import STORE -from orvex.contracts.store.kit import STORE_KIT +from orvex.contracts.store.kit import ISOLATED, STORE_KIT from orvex.contracts.store.kit_codes import code_cases @@ -33,7 +33,7 @@ def test_every_declared_code_has_its_case_in_the_kit() -> None: def test_a_code_declared_without_its_case_refuses_the_kit() -> None: widened = _with_errors("held", ("STORE_BUSY",)) with pytest.raises(InterfaceContractError) as caught: - code_cases(widened) # type: ignore[arg-type] + code_cases(widened, ISOLATED) # type: ignore[arg-type] assert caught.value.code == "KIT_CODE_UNPROVEN" assert "('held', 'STORE_BUSY')" in caught.value.message @@ -41,6 +41,6 @@ def test_a_code_declared_without_its_case_refuses_the_kit() -> None: def test_a_case_of_a_code_no_longer_declared_refuses_the_kit() -> None: narrowed = _with_errors("json_frame", ()) with pytest.raises(InterfaceContractError) as caught: - code_cases(narrowed) # type: ignore[arg-type] + code_cases(narrowed, ISOLATED) # type: ignore[arg-type] assert caught.value.code == "KIT_CODE_UNPROVEN" assert "('json_frame', 'STORE_TYPE_UNSUPPORTED')" in caught.value.message diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index 1357c436..d01b3308 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -238,7 +238,7 @@ contexts = [ "python-tests-3.14-part-8", "python-tests-3.14-part-9", ] -digest = "f87e8bad813dbf9e" +digest = "d13ee688e2a5c3bb" pythons = ["3.15", "3.14"] setup = [ "python -m pip install --quiet pip==26.3", @@ -762,7 +762,7 @@ job = "standalone-store" kind = "check" required = false contexts = ["standalone-store"] -digest = "f3100f3e79efae0a" +digest = "f94090838840ae0e" setup = [ 'dir="$RUNNER_TEMP/postgres-tls"', "python -m pip install --quiet pip==26.3",