From 95c00b093edbb4fb828ebd8e84ba95cf20473435 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 00:26:50 -0400 Subject: [PATCH 1/3] ci(repo): the threat-model check, required on every pull request (#244) threat_model.py (standard library) refuses a stale inventory (R1, R2), a surface without a section or a STRIDE category (R3, R4), and a row with an empty, unknown, duplicate, missing or malformed field (R5-R7); scoped workflow with changes/aggregate, required context, CI alert watch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .github/workflows/ci-alerts.yml | 1 + .github/workflows/scripts/ci_alerts.toml | 7 + .../workflows/scripts/test_threat_model.py | 177 ++++++++++++++++ .github/workflows/scripts/threat_model.py | 189 ++++++++++++++++++ .github/workflows/threat-model.yml | 79 ++++++++ docs/process/engineering.md | 1 + tools/ci/checks.toml | 42 ++++ tools/repo/main-ruleset.json | 4 + 8 files changed, 500 insertions(+) create mode 100644 .github/workflows/scripts/test_threat_model.py create mode 100644 .github/workflows/scripts/threat_model.py create mode 100644 .github/workflows/threat-model.yml diff --git a/.github/workflows/ci-alerts.yml b/.github/workflows/ci-alerts.yml index e31ad89f..3ef63c7d 100644 --- a/.github/workflows/ci-alerts.yml +++ b/.github/workflows/ci-alerts.yml @@ -25,6 +25,7 @@ on: - property-nightly - ossie-upstream - ci-alerts-selftest + - threat-model types: [completed] schedule: - cron: "41 */6 * * *" diff --git a/.github/workflows/scripts/ci_alerts.toml b/.github/workflows/scripts/ci_alerts.toml index cd102aa4..08cd0438 100644 --- a/.github/workflows/scripts/ci_alerts.toml +++ b/.github/workflows/scripts/ci_alerts.toml @@ -96,3 +96,10 @@ workflow = "ci-alerts-selftest.yml" events = ["workflow_dispatch"] required_jobs = ["selftest"] cadence = "none" + +[[watch]] +# Whole on every push to main (#244): the threat model stays current with every surface. +workflow = "threat-model.yml" +events = ["push"] +required_jobs = ["threat-model-changes", "model", "threat-model"] +cadence = "push" diff --git a/.github/workflows/scripts/test_threat_model.py b/.github/workflows/scripts/test_threat_model.py new file mode 100644 index 00000000..c7388b1f --- /dev/null +++ b/.github/workflows/scripts/test_threat_model.py @@ -0,0 +1,177 @@ +"""Tests of the threat-model check (#244): a valid model passes, and each rejection case R1-R7 +fails with its code, file and line.""" + +from __future__ import annotations + +import tempfile +import time +import unittest +from pathlib import Path + +from threat_model import CATEGORIES, check + +ROWS = "\n".join( + f"| TM-API-{n} | {category} | A threat. | security.md §2 | tests/test_api.py |" + for n, category in enumerate(CATEGORIES, start=1) +) +MODEL = f"""# Threat model + +```inventory +http /a api +http /b api +package one api +package two internal: not reachable yet +contract alpha api +contract beta api +contract http-api api +``` + +## Surface: api + +| ID | Category | Threat | Rule | Verification | +|---|---|---|---|---| +{ROWS} +""" + + +def repository(root: Path, model: str = MODEL) -> Path: + """A minimal repository: two paths, two packages, two contracts, the rule documents.""" + (root / "contracts/http-api/v1").mkdir(parents=True) + (root / "contracts/http-api/v1/openapi.yaml").write_text( + 'openapi: 3.1.0\npaths:\n /a:\n get: {}\n "/b":\n get: {}\ncomponents: {}\n' + ) + for folder in ( + "packages/one", + "packages/two", + "contracts/alpha", + "contracts/beta", + "contracts/_example", + "docs/adr", + "docs/architecture", + "docs/product", + "tests", + ): + (root / folder).mkdir(parents=True, exist_ok=True) + (root / "docs/adr/0027-two-data-routes.md").write_text("# ADR-0027\n") + (root / "docs/architecture/security.md").write_text("# Security\n\n## 2. Identity\n") + (root / "docs/product/laws.md").write_text("# Laws\n\n### Law 3 — Security by design\n") + (root / "docs/roadmap.md").write_text("| P3-01 | Security scenarios | conformance |\n") + (root / "tests/test_api.py").write_text("") + (root / "docs/architecture/threat-model.md").write_text(model) + return root + + +class ThreatModelTests(unittest.TestCase): + def setUp(self) -> None: + self.folder = tempfile.TemporaryDirectory() + self.addCleanup(self.folder.cleanup) + self.root = Path(self.folder.name) + + def problems(self, model: str = MODEL) -> list[str]: + return check(repository(self.root, model)) + + def assert_refused(self, model: str, code: str, line: int, needle: str) -> None: + found = self.problems(model) + self.assertTrue( + any( + p.startswith(f"{code} docs/architecture/threat-model.md:{line}:") and needle in p + for p in found + ), + found, + ) + + def test_a_valid_model_passes(self) -> None: + self.assertEqual(self.problems(), []) + + def test_every_allowed_form_passes(self) -> None: + model = MODEL.replace( + "security.md §2 | tests/test_api.py |", "ADR-0027 | roadmap:P3-01 |", 1 + ).replace("| security.md §2 | tests/test_api.py |", "| laws.md Law 3 | gap: no test |", 1) + self.assertEqual(self.problems(model), []) + + def test_r1_a_path_package_or_contract_missing_from_the_inventory(self) -> None: + for item in ("http /b api\n", "package one api\n", "contract beta api\n"): + with self.subTest(item=item): + root = Path(tempfile.mkdtemp(dir=self.folder.name)) + found = check(repository(root, MODEL.replace(item, ""))) + name = item.split()[1] + self.assertTrue( + any(p.startswith("THREAT_MODEL_STALE ") and name in p for p in found), found + ) + + def test_r2_an_inventory_item_that_no_longer_exists(self) -> None: + model = MODEL.replace("contract beta api\n", "contract beta api\npackage gone api\n") + self.assert_refused(model, "THREAT_MODEL_STALE", 10, "gone") + + def test_r3_an_inventory_item_naming_a_surface_without_a_section(self) -> None: + self.assert_refused( + MODEL.replace("http /b api", "http /b lake"), "THREAT_MODEL_INVALID", 5, "lake" + ) + + def test_r4_a_surface_missing_a_category(self) -> None: + model = "\n".join(line for line in MODEL.splitlines() if "| Repudiation |" not in line) + self.assert_refused(model + "\n", "THREAT_MODEL_INVALID", 13, "Repudiation") + + def test_r5_empty_rule_or_verification_unknown_category_or_duplicate_id(self) -> None: + cases = { + "empty rule": ( + MODEL.replace( + "| security.md §2 | tests/test_api.py |", "| | tests/test_api.py |", 1 + ), + "Rule", + ), + "empty verification": ( + MODEL.replace("| security.md §2 | tests/test_api.py |", "| security.md §2 | |", 1), + "Verification", + ), + "unknown category": (MODEL.replace("| Spoofing |", "| Phishing |", 1), "Phishing"), + "duplicate id": (MODEL.replace("TM-API-2 ", "TM-API-1 ", 1), "TM-API-1"), + } + for name, (model, needle) in cases.items(): + with self.subTest(name): + found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), model)) + self.assertTrue( + any(p.startswith("THREAT_MODEL_INVALID ") and needle in p for p in found), found + ) + + def test_r6_a_rule_naming_a_heading_or_adr_that_does_not_exist(self) -> None: + for rule in ("security.md §9", "ADR-0099", "laws.md Law 42", "elsewhere"): + with self.subTest(rule=rule): + model = MODEL.replace("security.md §2", rule, 1) + found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), model)) + self.assertTrue( + any(p.startswith("THREAT_MODEL_INVALID ") and rule in p for p in found), found + ) + + def test_r7_a_verification_path_that_does_not_exist_or_of_no_allowed_form(self) -> None: + for value in ("tests/missing.py", "roadmap:P9-99", "gap:", "tested elsewhere"): + with self.subTest(value=value): + model = MODEL.replace("tests/test_api.py", value, 1) + found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), model)) + self.assertTrue( + any(p.startswith("THREAT_MODEL_INVALID ") and value in p for p in found), found + ) + + def test_r7_n_a_only_on_a_category_with_no_other_row(self) -> None: + alone = MODEL.replace( + "| security.md §2 | tests/test_api.py |", "| security.md §2 | n/a: read only |", 1 + ) + self.assertEqual(self.problems(alone), []) + beside = alone.replace( + "| TM-API-6 |", + "| TM-API-7 | Spoofing | Another. | security.md §2 | tests/test_api.py |\n| TM-API-6 |", + ) + found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), beside)) + self.assertTrue( + any(p.startswith("THREAT_MODEL_INVALID ") and "n/a" in p for p in found), found + ) + + def test_the_repository_model_passes_in_under_five_seconds(self) -> None: + root = Path(__file__).resolve().parents[3] + start = time.monotonic() + self.assertEqual(check(root), []) + self.assertLess(time.monotonic() - start, 5) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/scripts/threat_model.py b/.github/workflows/scripts/threat_model.py new file mode 100644 index 00000000..82c55623 --- /dev/null +++ b/.github/workflows/scripts/threat_model.py @@ -0,0 +1,189 @@ +"""The threat-model check (#244): `docs/architecture/threat-model.md` lists every HTTP path, +package and contract directory, covers every STRIDE category per surface, and every row names a +rule that exists and a verification of an allowed form. Standard library only; fails closed. + +Usage: python3 threat_model.py [repository root]. Exits 1 printing ` :: `. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +MODEL = "docs/architecture/threat-model.md" +OPENAPI = "contracts/http-api/v1/openapi.yaml" +CATEGORIES = ( + "Spoofing", + "Tampering", + "Repudiation", + "Information disclosure", + "Denial of service", + "Elevation of privilege", +) +STALE = "THREAT_MODEL_STALE" +INVALID = "THREAT_MODEL_INVALID" +ROW = re.compile(r"^\| (TM-[A-Z]+-[0-9]+) \|(.*)\|\s*$") +SECTION = re.compile(r"^## Surface: (\S+)\s*$") +ADR = re.compile(r"^ADR-([0-9]{4})$") +SECURITY = re.compile(r"^security\.md §([0-9]+(?:\.[0-9]+)*)$") +LAW = re.compile(r"^laws\.md Law ([0-9]+)$") +ROADMAP = re.compile(r"^roadmap:(P[0-9]+-[0-9]+[a-z]?)$") +REASONED = re.compile(r"^(gap|n/a): \S") + + +def http_paths(root: Path) -> set[str]: + """The keys of the OpenAPI document's top-level `paths` mapping, read without a YAML parser: + the two-space-indented keys between `paths:` and the next top-level key.""" + paths: set[str] = set() + inside = False + for line in (root / OPENAPI).read_text(encoding="utf-8").splitlines(): + if re.match(r"^\S", line): + inside = line.rstrip() == "paths:" + continue + if inside and (found := re.match(r"^ (['\"]?)(/[^'\":]*)\1:\s*$", line)): + paths.add(found.group(2)) + return paths + + +def present(root: Path) -> set[tuple[str, str]]: + """Every item the inventory must list: (kind, name).""" + items = {("http", path) for path in http_paths(root)} + items |= {("package", d.name) for d in (root / "packages").iterdir() if d.is_dir()} + items |= { + ("contract", d.name) + for d in (root / "contracts").iterdir() + if d.is_dir() and d.name != "_example" + } + return items + + +def headings(path: Path, pattern: str) -> set[str]: + return set(re.findall(pattern, path.read_text(encoding="utf-8"), re.MULTILINE)) + + +def rule_problem(root: Path, rule: str) -> str | None: + if found := SECURITY.match(rule): + known = headings(root / "docs/architecture/security.md", r"^#+ ([0-9]+(?:\.[0-9]+)*)\.? ") + return None if found.group(1) in known else f"Rule {rule!r} names no security.md heading" + if found := ADR.match(rule): + exists = any((root / "docs/adr").glob(f"{found.group(1)}-*.md")) + return None if exists else f"Rule {rule!r} names no ADR file" + if found := LAW.match(rule): + known = headings(root / "docs/product/laws.md", r"^### Law ([0-9]+) ") + return None if found.group(1) in known else f"Rule {rule!r} names no law" + return f"Rule {rule!r} is not a security.md section, an ADR or a law" + + +def verification_problem(root: Path, value: str, alone: bool) -> str | None: + if found := ROADMAP.match(value): + known = headings(root / "docs/roadmap.md", r"^\| (P[0-9]+-[0-9]+[a-z]?) \|") + return None if found.group(1) in known else f"Verification {value!r} names no roadmap ID" + if REASONED.match(value): + if value.startswith("n/a") and not alone: + return f"Verification {value!r}: n/a only on a category with no other row" + return None + if value.startswith(("gap:", "n/a:", "roadmap:")): + return f"Verification {value!r} has no reason or no valid ID" + resolved = (root / value).resolve() + if root.resolve() in resolved.parents and resolved.exists(): + return None + return f"Verification {value!r} is not an existing path, a roadmap ID, a gap or n/a" + + +def check(root: Path) -> list[str]: + """Every problem of the model at `root`, each ` :: `.""" + lines = (root / MODEL).read_text(encoding="utf-8").splitlines() + problems: list[str] = [] + + def report(code: str, number: int, why: str) -> None: + problems.append(f"{code} {MODEL}:{number}: {why}") + + inventory: dict[tuple[str, str], tuple[int, str]] = {} + sections: dict[str, int] = {} + rows: dict[str, list[tuple[int, str, str, str, str]]] = {} + fence = False + current: str | None = None + for number, line in enumerate(lines, start=1): + if line.strip() == "```inventory": + fence = True + continue + if fence: + if line.strip() == "```": + fence = False + continue + parts = line.split(maxsplit=2) + if len(parts) < 3 or parts[0] not in {"http", "package", "contract"}: + report(INVALID, number, f"inventory line {line!r} is not ' '") + continue + inventory[(parts[0], parts[1])] = (number, parts[2]) + continue + if found := SECTION.match(line): + current = found.group(1) + sections[current] = number + rows[current] = [] + continue + if found := ROW.match(line): + cells = [cell.strip() for cell in found.group(2).split("|")] + if current is None or len(cells) != 4: + report( + INVALID, number, f"row {found.group(1)} is outside a surface or not 5 columns" + ) + continue + rows[current].append((number, found.group(1), cells[0], cells[2], cells[3])) + if fence: + report(INVALID, len(lines), "the inventory block is not closed") + + for kind, name in sorted(present(root) - inventory.keys()): + report(STALE, 1, f"{kind} {name} is missing from the inventory") + for (kind, name), (number, _) in sorted(inventory.items()): + if (kind, name) not in present(root): + report(STALE, number, f"{kind} {name} no longer exists") + for (kind, name), (number, surface) in sorted(inventory.items(), key=lambda i: i[1][0]): + if not surface.startswith("internal: ") and surface not in sections: + report( + INVALID, number, f"{kind} {name} names surface {surface!r}, which has no section" + ) + if surface.startswith("internal:") and not surface[len("internal:") :].strip(): + report(INVALID, number, f"{kind} {name} is internal with no reason") + + seen: dict[str, int] = {} + for surface, found_rows in rows.items(): + counts = dict.fromkeys(CATEGORIES, 0) + for _, _, category, _, _ in found_rows: + if category in counts: + counts[category] += 1 + for category in CATEGORIES: + if not counts[category]: + report(INVALID, sections[surface], f"surface {surface} has no {category} row") + for number, row_id, category, rule, value in found_rows: + if row_id in seen: + report(INVALID, number, f"{row_id} duplicates the ID of line {seen[row_id]}") + seen.setdefault(row_id, number) + if category not in CATEGORIES: + report(INVALID, number, f"{row_id}: unknown category {category!r}") + if not rule: + report(INVALID, number, f"{row_id}: empty Rule") + elif why := rule_problem(root, rule): + report(INVALID, number, f"{row_id}: {why}") + if not value: + report(INVALID, number, f"{row_id}: empty Verification") + elif why := verification_problem(root, value, counts.get(category, 0) == 1): + report(INVALID, number, f"{row_id}: {why}") + return problems + + +def main(argv: list[str]) -> int: + root = Path(argv[1]) if len(argv) > 1 else Path.cwd() + try: + problems = check(root) + except (OSError, UnicodeDecodeError) as error: + print(f"{INVALID} {MODEL}:1: cannot read the repository: {error}") + return 1 + for problem in problems: + print(problem) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/workflows/threat-model.yml b/.github/workflows/threat-model.yml new file mode 100644 index 00000000..dc021e34 --- /dev/null +++ b/.github/workflows/threat-model.yml @@ -0,0 +1,79 @@ +# The threat-model check (#244): `docs/architecture/threat-model.md` lists every HTTP path, +# package and contract directory, covers every STRIDE category per surface, and every row names a +# rule that exists and a verification of an allowed form (`scripts/threat_model.py`, standard +# library only). Scoped (#267): it applies when the model, what it inventories or what its rows +# cite changes, and always on main. +name: threat-model + +on: + pull_request: + push: + branches: [main] + +# A newer commit on a pull request cancels the pull request's older runs (#386). Pushes to one +# branch (main) share a group: a running one is never cancelled, and only the newest waiting +# one is kept, since it tests everything the older ones would (#391). +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + # The paths that make this check apply (#267), read by changes.py and aggregate.py: the model + # and the check; what the inventory lists (the HTTP contract, packages, contracts); and what + # rows cite (rule documents, tests and scenarios). + RELEVANT_PATHS: | + .github/workflows/threat-model.yml + .github/workflows/scripts/threat_model.py + docs/** + contracts/** + packages/** + conformance/** + tests/** + +jobs: + # Whether this run changes a relevant path (#267): `changes.py` fails closed. + changes: + name: threat-model-changes + runs-on: ubuntu-latest + outputs: + relevant: ${{ steps.changes.outputs.relevant }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - id: changes + env: + EVENT: ${{ github.event_name }} + BASE: ${{ github.event.pull_request.base.sha }} + HEAD: ${{ github.event.pull_request.head.sha }} + run: python3 .github/workflows/scripts/changes.py + + model: + needs: changes + if: needs.changes.outputs.relevant == 'true' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 .github/workflows/scripts/threat_model.py + + threat-model: + needs: [changes, model] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 .github/workflows/scripts/aggregate.py + env: + CHECK: threat-model + CHANGES: ${{ needs.changes.result }} + RELEVANT: ${{ needs.changes.outputs.relevant }} + RESULT: ${{ needs.model.result }} diff --git a/docs/process/engineering.md b/docs/process/engineering.md index 4132773a..08c8caa4 100644 --- a/docs/process/engineering.md +++ b/docs/process/engineering.md @@ -114,6 +114,7 @@ A check becomes **required** (blocks merging) from the issue that creates it; fr | Contracts | JSON Schema and OpenAPI validation; generated types current | P0-05 | P0 | | Semantic corpus | corpus runner, reporting per case; from P2-27a the `semantic-corpus` job runs `python -m conformance.runners.semantic_corpus --through both` (library and HTTP API, JSON and Arrow), failing on any failed case, any difference between the two ways, or any finding | P0-11 | Reported from P0; **gate 21/21 from P2-27** | | Security conformance | scenarios runner; from P3-13 the `security-scenarios` check (scoped, whole on `main`) runs `python -m conformance.runners.security_scenarios` on the managed lake, an existing DuckLake on SQLite and PostgreSQL catalogs, and ClickHouse 26.10 and 26.8 LTS (`--require-clickhouse 26.10,26.8`), failing on any failed, stale or missing execution and uploading its JSON report | P3-13 | **Required from P3-13** | +| Threat model | `threat-model` check (scoped, whole on `main`): `python3 .github/workflows/scripts/threat_model.py` checks that `docs/architecture/threat-model.md` lists every HTTP path, package and contract directory, covers every STRIDE category per surface, and gives every row an existing rule and an allowed verification | #244 | P3 | | End-to-end | browser tests on main flows, both runtime modes on Linux | P6-21 | P6 | | Accessibility | automated checks on every route | P6-21 | P6 | | Operating-system matrix | local mode on Linux, Windows, macOS | P9-10 | P9 | diff --git a/tools/ci/checks.toml b/tools/ci/checks.toml index b13846a9..b889b7e3 100644 --- a/tools/ci/checks.toml +++ b/tools/ci/checks.toml @@ -721,3 +721,45 @@ name = "semantic-corpus-clickhouse-26.8" run = "python -m conformance.runners.semantic_corpus --through clickhouse --switch-on" local = false reason = "needs a ClickHouse 26.8 LTS server (CORPUS_CLICKHOUSE), started by the job from its release archive" + +[[job]] +workflow = "threat-model.yml" +job = "changes" +kind = "check" +digest = "3426e21cce9cb74c" +required = false +contexts = ["threat-model-changes"] +setup = [] +[[job.check]] +name = "changes-threat-model" +run = "python3 .github/workflows/scripts/changes.py" +local = false +reason = "reads the pull request's base and head commits in CI; its rules are tested by governance-scripts" + +[[job]] +workflow = "threat-model.yml" +job = "model" +kind = "scoped" +reason = "runs only when the model, what its inventory lists or what its rows cite changes; the check itself runs locally (threat-model)" +digest = "5fa23f40f2a05776" +paths = [".github/workflows/threat-model.yml", ".github/workflows/scripts/threat_model.py", "docs/**", "contracts/**", "packages/**", "conformance/**", "tests/**"] +setup = [] +[[job.check]] +name = "threat-model" +run = "python3 .github/workflows/scripts/threat_model.py" +local = true + +[[job]] +workflow = "threat-model.yml" +job = "threat-model" +kind = "check" +digest = "980758626838ff5c" +# Required from #244 (engineering.md §8): the threat model stays current with every surface. +required = true +contexts = ["threat-model"] +setup = [] +[[job.check]] +name = "threat-model-aggregate" +run = "python3 .github/workflows/scripts/aggregate.py" +local = false +reason = "reports the result of this run's CI jobs; its rules are tested by governance-scripts" diff --git a/tools/repo/main-ruleset.json b/tools/repo/main-ruleset.json index a8a62eb8..bee39901 100644 --- a/tools/repo/main-ruleset.json +++ b/tools/repo/main-ruleset.json @@ -106,6 +106,10 @@ { "context": "security-scenarios", "integration_id": 15368 + }, + { + "context": "threat-model", + "integration_id": 15368 } ] } From e7f3598563676d7a119e50704a9613b9c0492542 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:08:05 -0400 Subject: [PATCH 2/3] ci(repo): the threat-model check reads every path and refuses what it cannot read (#244) Reviewer findings 1-5: the OpenAPI reader keeps comments, inline values and colons in paths and refuses an unreadable key; unmatched table lines, a second section and a repeated inventory item are refused; a Verification is a test file or a scenario path inside the repository; tests assert code, line and reason; references are read once. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .../workflows/scripts/test_threat_model.py | 292 +++++++++++++----- .github/workflows/scripts/threat_model.py | 159 +++++++--- 2 files changed, 330 insertions(+), 121 deletions(-) diff --git a/.github/workflows/scripts/test_threat_model.py b/.github/workflows/scripts/test_threat_model.py index c7388b1f..9315b36e 100644 --- a/.github/workflows/scripts/test_threat_model.py +++ b/.github/workflows/scripts/test_threat_model.py @@ -1,19 +1,24 @@ """Tests of the threat-model check (#244): a valid model passes, and each rejection case R1-R7 -fails with its code, file and line.""" +fails with its code, file, line and reason.""" from __future__ import annotations +import io import tempfile import time import unittest +from contextlib import redirect_stdout from pathlib import Path -from threat_model import CATEGORIES, check +from threat_model import CATEGORIES, OPENAPI, check, main +from threat_model import MODEL as MODEL_PATH ROWS = "\n".join( f"| TM-API-{n} | {category} | A threat. | security.md §2 | tests/test_api.py |" for n, category in enumerate(CATEGORIES, start=1) ) +# Line numbers: the inventory fence is line 3, its items lines 4-10, the section line 13, and the +# rows TM-API-1..6 lines 17-22. MODEL = f"""# Threat model ```inventory @@ -32,16 +37,15 @@ |---|---|---|---|---| {ROWS} """ +OPENAPI_TEXT = 'openapi: 3.1.0\npaths:\n /a:\n get: {}\n "/b":\n get: {}\ncomponents: {}\n' -def repository(root: Path, model: str = MODEL) -> Path: - """A minimal repository: two paths, two packages, two contracts, the rule documents.""" +def repository(root: Path, model: str = MODEL, openapi: str = OPENAPI_TEXT) -> Path: + """A minimal repository: two paths, two packages, three contracts, the rule documents.""" (root / "contracts/http-api/v1").mkdir(parents=True) - (root / "contracts/http-api/v1/openapi.yaml").write_text( - 'openapi: 3.1.0\npaths:\n /a:\n get: {}\n "/b":\n get: {}\ncomponents: {}\n' - ) + (root / OPENAPI).write_text(openapi) for folder in ( - "packages/one", + "packages/one/tests", "packages/two", "contracts/alpha", "contracts/beta", @@ -50,6 +54,7 @@ def repository(root: Path, model: str = MODEL) -> Path: "docs/architecture", "docs/product", "tests", + "conformance/security/scenarios/rows", ): (root / folder).mkdir(parents=True, exist_ok=True) (root / "docs/adr/0027-two-data-routes.md").write_text("# ADR-0027\n") @@ -57,113 +62,250 @@ def repository(root: Path, model: str = MODEL) -> Path: (root / "docs/product/laws.md").write_text("# Laws\n\n### Law 3 — Security by design\n") (root / "docs/roadmap.md").write_text("| P3-01 | Security scenarios | conformance |\n") (root / "tests/test_api.py").write_text("") - (root / "docs/architecture/threat-model.md").write_text(model) + (root / "packages/one/tests/test_one.py").write_text("") + (root / "README.md").write_text("") + (root / MODEL_PATH).write_text(model) return root +def first_row(rule: str = "security.md §2", value: str = "tests/test_api.py") -> str: + """MODEL with TM-API-1's Rule and Verification replaced.""" + return MODEL.replace("security.md §2 | tests/test_api.py |", f"{rule} | {value} |", 1) + + class ThreatModelTests(unittest.TestCase): def setUp(self) -> None: self.folder = tempfile.TemporaryDirectory() self.addCleanup(self.folder.cleanup) - self.root = Path(self.folder.name) - def problems(self, model: str = MODEL) -> list[str]: - return check(repository(self.root, model)) + def problems(self, model: str = MODEL, openapi: str = OPENAPI_TEXT) -> list[str]: + root = Path(tempfile.mkdtemp(dir=self.folder.name)) + return check(repository(root, model, openapi)) - def assert_refused(self, model: str, code: str, line: int, needle: str) -> None: - found = self.problems(model) - self.assertTrue( - any( - p.startswith(f"{code} docs/architecture/threat-model.md:{line}:") and needle in p - for p in found - ), - found, - ) + def assert_only(self, found: list[str], expected: str) -> None: + """`found` holds exactly one problem, `expected`.""" + self.assertEqual(found, [expected]) + + def model_problem(self, code: str, line: int, why: str) -> str: + return f"{code} {MODEL_PATH}:{line}: {why}" def test_a_valid_model_passes(self) -> None: self.assertEqual(self.problems(), []) def test_every_allowed_form_passes(self) -> None: - model = MODEL.replace( - "security.md §2 | tests/test_api.py |", "ADR-0027 | roadmap:P3-01 |", 1 - ).replace("| security.md §2 | tests/test_api.py |", "| laws.md Law 3 | gap: no test |", 1) - self.assertEqual(self.problems(model), []) + values = ( + ("ADR-0027", "roadmap:P3-01"), + ("laws.md Law 3", "gap: no test yet"), + ("security.md §2", "packages/one/tests/test_one.py"), + ("security.md §2", "conformance/security/scenarios/rows"), + ("security.md §2", "n/a: does not apply"), + ) + for rule, value in values: + with self.subTest(value=value): + self.assertEqual(self.problems(first_row(rule, value)), []) - def test_r1_a_path_package_or_contract_missing_from_the_inventory(self) -> None: + def test_r1_an_item_missing_from_the_inventory_is_reported_at_the_fence(self) -> None: for item in ("http /b api\n", "package one api\n", "contract beta api\n"): with self.subTest(item=item): - root = Path(tempfile.mkdtemp(dir=self.folder.name)) - found = check(repository(root, MODEL.replace(item, ""))) - name = item.split()[1] - self.assertTrue( - any(p.startswith("THREAT_MODEL_STALE ") and name in p for p in found), found + kind, name = item.split()[:2] + self.assert_only( + self.problems(MODEL.replace(item, "")), + self.model_problem( + "THREAT_MODEL_STALE", 3, f"{kind} {name} is missing from the inventory" + ), ) def test_r2_an_inventory_item_that_no_longer_exists(self) -> None: - model = MODEL.replace("contract beta api\n", "contract beta api\npackage gone api\n") - self.assert_refused(model, "THREAT_MODEL_STALE", 10, "gone") + model = MODEL.replace( + "contract http-api api\n", "contract http-api api\npackage gone api\n" + ) + self.assert_only( + self.problems(model), + self.model_problem("THREAT_MODEL_STALE", 11, "package gone no longer exists"), + ) def test_r3_an_inventory_item_naming_a_surface_without_a_section(self) -> None: - self.assert_refused( - MODEL.replace("http /b api", "http /b lake"), "THREAT_MODEL_INVALID", 5, "lake" + self.assert_only( + self.problems(MODEL.replace("http /b api", "http /b lake")), + self.model_problem( + "THREAT_MODEL_INVALID", 5, "http /b names surface 'lake', which has no section" + ), + ) + + def test_r3_an_internal_item_needs_a_reason(self) -> None: + self.assert_only( + self.problems(MODEL.replace("internal: not reachable yet", "internal:")), + self.model_problem("THREAT_MODEL_INVALID", 7, "package two is internal with no reason"), ) def test_r4_a_surface_missing_a_category(self) -> None: - model = "\n".join(line for line in MODEL.splitlines() if "| Repudiation |" not in line) - self.assert_refused(model + "\n", "THREAT_MODEL_INVALID", 13, "Repudiation") + model = ( + "\n".join(line for line in MODEL.splitlines() if "| Repudiation |" not in line) + "\n" + ) + self.assert_only( + self.problems(model), + self.model_problem("THREAT_MODEL_INVALID", 13, "surface api has no Repudiation row"), + ) - def test_r5_empty_rule_or_verification_unknown_category_or_duplicate_id(self) -> None: - cases = { - "empty rule": ( - MODEL.replace( - "| security.md §2 | tests/test_api.py |", "| | tests/test_api.py |", 1 - ), - "Rule", - ), - "empty verification": ( - MODEL.replace("| security.md §2 | tests/test_api.py |", "| security.md §2 | |", 1), - "Verification", - ), - "unknown category": (MODEL.replace("| Spoofing |", "| Phishing |", 1), "Phishing"), - "duplicate id": (MODEL.replace("TM-API-2 ", "TM-API-1 ", 1), "TM-API-1"), - } - for name, (model, needle) in cases.items(): - with self.subTest(name): - found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), model)) + def test_r5_an_empty_rule_or_verification(self) -> None: + self.assert_only( + self.problems(first_row(rule="")), + self.model_problem("THREAT_MODEL_INVALID", 17, "TM-API-1: empty Rule"), + ) + self.assert_only( + self.problems(first_row(value="")), + self.model_problem("THREAT_MODEL_INVALID", 17, "TM-API-1: empty Verification"), + ) + + def test_r5_an_unknown_category_or_a_duplicate_id(self) -> None: + found = self.problems( + MODEL.replace("| TM-API-1 | Spoofing |", "| TM-API-1 | Phishing |", 1) + ) + self.assertIn( + self.model_problem("THREAT_MODEL_INVALID", 17, "TM-API-1: unknown category 'Phishing'"), + found, + ) + self.assert_only( + self.problems(MODEL.replace("TM-API-2 ", "TM-API-1 ", 1)), + self.model_problem("THREAT_MODEL_INVALID", 18, "TM-API-1 duplicates the ID of line 17"), + ) + + def test_r5_a_table_line_that_is_not_a_row_is_refused_not_skipped(self) -> None: + for bad in ("| TM-Api-7 |", "|TM-API-7|", "| TM-API-7a |"): + with self.subTest(bad=bad): + line = f"{bad} Spoofing | A threat. | security.md §2 | tests/test_api.py |" + found = self.problems(MODEL + line + "\n") + self.assertEqual(len(found), 1, found) self.assertTrue( - any(p.startswith("THREAT_MODEL_INVALID ") and needle in p for p in found), found + found[0].startswith(f"THREAT_MODEL_INVALID {MODEL_PATH}:23: table line"), found ) - def test_r6_a_rule_naming_a_heading_or_adr_that_does_not_exist(self) -> None: - for rule in ("security.md §9", "ADR-0099", "laws.md Law 42", "elsewhere"): + def test_r5_a_second_section_for_one_surface(self) -> None: + model = MODEL + "\n## Surface: api\n" + self.assert_only( + self.problems(model), + self.model_problem("THREAT_MODEL_INVALID", 24, "surface api has a second section"), + ) + + def test_r5_an_inventory_item_listed_twice(self) -> None: + model = MODEL.replace("http /a api\n", "http /a api\nhttp /a api\n") + self.assert_only( + self.problems(model), + self.model_problem( + "THREAT_MODEL_INVALID", 5, "http /a is listed twice in the inventory" + ), + ) + + def test_r5_a_malformed_or_unclosed_inventory_and_a_row_outside_a_surface(self) -> None: + self.assertIn( + self.model_problem( + "THREAT_MODEL_INVALID", + 4, + "inventory line 'path /a api' is not ' '", + ), + self.problems(MODEL.replace("http /a api", "path /a api")), + ) + unclosed = MODEL.replace("contract http-api api\n```\n", "contract http-api api\n") + self.assertIn( + self.model_problem( + "THREAT_MODEL_INVALID", + len(unclosed.splitlines()), + "the inventory block is not closed", + ), + self.problems(unclosed), + ) + outside = MODEL.replace( + "# Threat model\n", "# Threat model\n| TM-API-9 | Spoofing | x | y | z |\n", 1 + ) + self.assertIn( + self.model_problem( + "THREAT_MODEL_INVALID", 2, "row TM-API-9 is outside a surface section" + ), + self.problems(outside), + ) + + def test_r6_a_rule_naming_nothing_that_exists(self) -> None: + cases = { + "security.md §9": "Rule 'security.md §9' names no security.md heading", + "ADR-0099": "Rule 'ADR-0099' names no ADR file", + "laws.md Law 42": "Rule 'laws.md Law 42' names no law", + "elsewhere": "Rule 'elsewhere' is not a security.md section, an ADR or a law", + } + for rule, why in cases.items(): with self.subTest(rule=rule): - model = MODEL.replace("security.md §2", rule, 1) - found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), model)) - self.assertTrue( - any(p.startswith("THREAT_MODEL_INVALID ") and rule in p for p in found), found + self.assert_only( + self.problems(first_row(rule=rule)), + self.model_problem("THREAT_MODEL_INVALID", 17, f"TM-API-1: {why}"), ) - def test_r7_a_verification_path_that_does_not_exist_or_of_no_allowed_form(self) -> None: - for value in ("tests/missing.py", "roadmap:P9-99", "gap:", "tested elsewhere"): + def test_r7_a_verification_of_no_allowed_form(self) -> None: + tail = "is not a test file, a scenario path, a roadmap ID, a gap or n/a" + cases = { + "tests/missing.py": f"Verification 'tests/missing.py' {tail}", + "README.md": f"Verification 'README.md' {tail}", + "tests": f"Verification 'tests' {tail}", + "../outside.py": f"Verification '../outside.py' {tail}", + "tested elsewhere": f"Verification 'tested elsewhere' {tail}", + "roadmap:P9-99": "Verification 'roadmap:P9-99' names no roadmap ID", + "gap:": "Verification 'gap:' has no reason or no valid ID", + } + for value, why in cases.items(): with self.subTest(value=value): - model = MODEL.replace("tests/test_api.py", value, 1) - found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), model)) - self.assertTrue( - any(p.startswith("THREAT_MODEL_INVALID ") and value in p for p in found), found + self.assert_only( + self.problems(first_row(value=value)), + self.model_problem("THREAT_MODEL_INVALID", 17, f"TM-API-1: {why}"), ) def test_r7_n_a_only_on_a_category_with_no_other_row(self) -> None: - alone = MODEL.replace( - "| security.md §2 | tests/test_api.py |", "| security.md §2 | n/a: read only |", 1 - ) - self.assertEqual(self.problems(alone), []) - beside = alone.replace( + beside = first_row(value="n/a: read only").replace( "| TM-API-6 |", "| TM-API-7 | Spoofing | Another. | security.md §2 | tests/test_api.py |\n| TM-API-6 |", ) - found = check(repository(Path(tempfile.mkdtemp(dir=self.folder.name)), beside)) + self.assert_only( + self.problems(beside), + self.model_problem( + "THREAT_MODEL_INVALID", + 17, + "TM-API-1: Verification 'n/a: read only': n/a only on a category with no other row", + ), + ) + + def test_openapi_paths_are_read_through_comments_inline_values_and_colons(self) -> None: + openapi = ( + "openapi: 3.1.0\n" + "paths: # the routes\n" + " /a: # first\n" + " get: {}\n" + "# a comment at column 0 inside the block\n" + "\n" + " '/b': {$ref: b.yaml}\n" + " /v1/x:run:\n" + " post: {}\n" + "components: {}\n" + ) + found = self.problems( + MODEL.replace("http /b api\n", "http /b api\nhttp /v1/x:run api\n"), openapi + ) + self.assertEqual(found, []) + + def test_an_unreadable_path_key_fails_closed(self) -> None: + openapi = "openapi: 3.1.0\npaths:\n /a:\n get: {}\n not-a-path:\n get: {}\n" + found = self.problems(MODEL.replace("http /b api\n", ""), openapi) + self.assertIn(f"THREAT_MODEL_INVALID {OPENAPI}:5: 'not-a-path:' is not a path key", found) + + def test_a_missing_or_empty_paths_mapping_fails_closed(self) -> None: + expected = f"THREAT_MODEL_INVALID {OPENAPI}:1: no top-level `paths:` mapping with a path" + for openapi in ("openapi: 3.1.0\ncomponents: {}\n", "openapi: 3.1.0\npaths: {}\n"): + with self.subTest(openapi=openapi): + self.assertIn(expected, self.problems(openapi=openapi)) + + def test_main_exits_1_when_the_repository_cannot_be_read(self) -> None: + output = io.StringIO() + with redirect_stdout(output): + code = main(["threat_model.py", str(Path(self.folder.name) / "missing")]) + self.assertEqual(code, 1) self.assertTrue( - any(p.startswith("THREAT_MODEL_INVALID ") and "n/a" in p for p in found), found + output.getvalue().startswith(f"THREAT_MODEL_INVALID {MODEL_PATH}:1: cannot read") ) def test_the_repository_model_passes_in_under_five_seconds(self) -> None: diff --git a/.github/workflows/scripts/threat_model.py b/.github/workflows/scripts/threat_model.py index 82c55623..50af4e9f 100644 --- a/.github/workflows/scripts/threat_model.py +++ b/.github/workflows/scripts/threat_model.py @@ -9,6 +9,7 @@ import re import sys +from dataclasses import dataclass from pathlib import Path MODEL = "docs/architecture/threat-model.md" @@ -24,31 +25,65 @@ STALE = "THREAT_MODEL_STALE" INVALID = "THREAT_MODEL_INVALID" ROW = re.compile(r"^\| (TM-[A-Z]+-[0-9]+) \|(.*)\|\s*$") +HEADER = re.compile(r"^\| ID \| Category \| Threat \| Rule \| Verification \|\s*$") +SEPARATOR = re.compile(r"^\|(?:-+\|)+\s*$") SECTION = re.compile(r"^## Surface: (\S+)\s*$") ADR = re.compile(r"^ADR-([0-9]{4})$") SECURITY = re.compile(r"^security\.md §([0-9]+(?:\.[0-9]+)*)$") LAW = re.compile(r"^laws\.md Law ([0-9]+)$") ROADMAP = re.compile(r"^roadmap:(P[0-9]+-[0-9]+[a-z]?)$") REASONED = re.compile(r"^(gap|n/a): \S") +# An OpenAPI path key at indent 2. The key ends at its last colon before the end of the line or a +# space, so a colon inside the path (`/x:run`) and an inline value (`/a: {...}`) are both read. +PATH_KEY = re.compile(r"^ (['\"]?)(/\S*?)\1:(?:\s.*)?$") +# The only places a Verification may point to: test files, and security scenario files or folders. +TEST_FILES = re.compile(r"^(?:tests/|packages/[a-z]+/tests/)\S+$") +SCENARIOS = "conformance/security/scenarios/" -def http_paths(root: Path) -> set[str]: - """The keys of the OpenAPI document's top-level `paths` mapping, read without a YAML parser: - the two-space-indented keys between `paths:` and the next top-level key.""" +@dataclass(frozen=True) +class References: + """What rows may cite, read once: security.md section numbers, laws and roadmap IDs.""" + + security: frozenset[str] + laws: frozenset[str] + roadmap: frozenset[str] + + +def strip_comment(line: str) -> str: + """A YAML line without its comment (a `#` at the start of the line or after a space).""" + return re.sub(r"(^|\s)#.*$", "", line).rstrip() + + +def http_paths(root: Path) -> tuple[set[str], list[str]]: + """The keys of the OpenAPI document's top-level `paths` mapping, read without a YAML parser, + and a problem for every indent-2 line inside it that is not a path key (fails closed).""" paths: set[str] = set() - inside = False - for line in (root / OPENAPI).read_text(encoding="utf-8").splitlines(): + problems: list[str] = [] + inside = found_block = False + text = (root / OPENAPI).read_text(encoding="utf-8") + for number, raw in enumerate(text.splitlines(), start=1): + line = strip_comment(raw) + if not line: + continue if re.match(r"^\S", line): - inside = line.rstrip() == "paths:" + inside = line == "paths:" + found_block = found_block or inside + continue + if not inside or not re.match(r"^ \S", line): continue - if inside and (found := re.match(r"^ (['\"]?)(/[^'\":]*)\1:\s*$", line)): - paths.add(found.group(2)) - return paths + if key := PATH_KEY.match(line): + paths.add(key.group(2)) + else: + problems.append(f"{INVALID} {OPENAPI}:{number}: {raw.strip()!r} is not a path key") + if not found_block or not paths: + problems.append(f"{INVALID} {OPENAPI}:1: no top-level `paths:` mapping with a path") + return paths, problems -def present(root: Path) -> set[tuple[str, str]]: +def present(root: Path, paths: set[str]) -> set[tuple[str, str]]: """Every item the inventory must list: (kind, name).""" - items = {("http", path) for path in http_paths(root)} + items = {("http", path) for path in paths} items |= {("package", d.name) for d in (root / "packages").iterdir() if d.is_dir()} items |= { ("contract", d.name) @@ -58,43 +93,59 @@ def present(root: Path) -> set[tuple[str, str]]: return items -def headings(path: Path, pattern: str) -> set[str]: - return set(re.findall(pattern, path.read_text(encoding="utf-8"), re.MULTILINE)) +def references(root: Path) -> References: + """The headings, laws and roadmap IDs rows may cite, each document read once.""" + + def found(path: str, pattern: str) -> frozenset[str]: + text = (root / path).read_text(encoding="utf-8") + return frozenset(re.findall(pattern, text, re.MULTILINE)) + + return References( + security=found("docs/architecture/security.md", r"^#+ ([0-9]+(?:\.[0-9]+)*)\.? "), + laws=found("docs/product/laws.md", r"^### Law ([0-9]+) "), + roadmap=found("docs/roadmap.md", r"^\| (P[0-9]+-[0-9]+[a-z]?) \|"), + ) -def rule_problem(root: Path, rule: str) -> str | None: +def rule_problem(root: Path, known: References, rule: str) -> str | None: if found := SECURITY.match(rule): - known = headings(root / "docs/architecture/security.md", r"^#+ ([0-9]+(?:\.[0-9]+)*)\.? ") - return None if found.group(1) in known else f"Rule {rule!r} names no security.md heading" + if found.group(1) in known.security: + return None + return f"Rule {rule!r} names no security.md heading" if found := ADR.match(rule): - exists = any((root / "docs/adr").glob(f"{found.group(1)}-*.md")) - return None if exists else f"Rule {rule!r} names no ADR file" + if any((root / "docs/adr").glob(f"{found.group(1)}-*.md")): + return None + return f"Rule {rule!r} names no ADR file" if found := LAW.match(rule): - known = headings(root / "docs/product/laws.md", r"^### Law ([0-9]+) ") - return None if found.group(1) in known else f"Rule {rule!r} names no law" + return None if found.group(1) in known.laws else f"Rule {rule!r} names no law" return f"Rule {rule!r} is not a security.md section, an ADR or a law" -def verification_problem(root: Path, value: str, alone: bool) -> str | None: +def verification_problem(root: Path, known: References, value: str, alone: bool) -> str | None: if found := ROADMAP.match(value): - known = headings(root / "docs/roadmap.md", r"^\| (P[0-9]+-[0-9]+[a-z]?) \|") - return None if found.group(1) in known else f"Verification {value!r} names no roadmap ID" + if found.group(1) in known.roadmap: + return None + return f"Verification {value!r} names no roadmap ID" if REASONED.match(value): if value.startswith("n/a") and not alone: return f"Verification {value!r}: n/a only on a category with no other row" return None if value.startswith(("gap:", "n/a:", "roadmap:")): return f"Verification {value!r} has no reason or no valid ID" - resolved = (root / value).resolve() - if root.resolve() in resolved.parents and resolved.exists(): + target = (root / value).resolve() + inside = root.resolve() in target.parents + if inside and TEST_FILES.match(value) and target.is_file(): return None - return f"Verification {value!r} is not an existing path, a roadmap ID, a gap or n/a" + if inside and value.startswith(SCENARIOS) and target.exists(): + return None + return f"Verification {value!r} is not a test file, a scenario path, a roadmap ID, a gap or n/a" def check(root: Path) -> list[str]: """Every problem of the model at `root`, each ` :: `.""" lines = (root / MODEL).read_text(encoding="utf-8").splitlines() - problems: list[str] = [] + paths, problems = http_paths(root) + known = references(root) def report(code: str, number: int, why: str) -> None: problems.append(f"{code} {MODEL}:{number}: {why}") @@ -103,10 +154,11 @@ def report(code: str, number: int, why: str) -> None: sections: dict[str, int] = {} rows: dict[str, list[tuple[int, str, str, str, str]]] = {} fence = False + fence_line = 0 current: str | None = None for number, line in enumerate(lines, start=1): if line.strip() == "```inventory": - fence = True + fence, fence_line = True, number continue if fence: if line.strip() == "```": @@ -116,36 +168,51 @@ def report(code: str, number: int, why: str) -> None: if len(parts) < 3 or parts[0] not in {"http", "package", "contract"}: report(INVALID, number, f"inventory line {line!r} is not ' '") continue + if (parts[0], parts[1]) in inventory: + report(INVALID, number, f"{parts[0]} {parts[1]} is listed twice in the inventory") + continue inventory[(parts[0], parts[1])] = (number, parts[2]) continue if found := SECTION.match(line): current = found.group(1) + if current in sections: + report(INVALID, number, f"surface {current} has a second section") + continue sections[current] = number rows[current] = [] continue - if found := ROW.match(line): - cells = [cell.strip() for cell in found.group(2).split("|")] - if current is None or len(cells) != 4: - report( - INVALID, number, f"row {found.group(1)} is outside a surface or not 5 columns" - ) - continue - rows[current].append((number, found.group(1), cells[0], cells[2], cells[3])) + row = ROW.match(line) + if current is None: + if row: + report(INVALID, number, f"row {row.group(1)} is outside a surface section") + continue + if not line.startswith("|") or HEADER.match(line) or SEPARATOR.match(line): + continue + cells = [cell.strip() for cell in row.group(2).split("|")] if row else [] + if not row or len(cells) != 4: + report( + INVALID, number, f"table line {line[:40]!r} is not a 5-column TM-- row" + ) + continue + rows[current].append((number, row.group(1), cells[0], cells[2], cells[3])) if fence: report(INVALID, len(lines), "the inventory block is not closed") + if not fence_line: + report(INVALID, 1, "the model has no inventory block") - for kind, name in sorted(present(root) - inventory.keys()): - report(STALE, 1, f"{kind} {name} is missing from the inventory") - for (kind, name), (number, _) in sorted(inventory.items()): - if (kind, name) not in present(root): + expected = present(root, paths) + for kind, name in sorted(expected - inventory.keys()): + report(STALE, fence_line or 1, f"{kind} {name} is missing from the inventory") + for (kind, name), (number, surface) in sorted(inventory.items(), key=lambda item: item[1][0]): + if (kind, name) not in expected: report(STALE, number, f"{kind} {name} no longer exists") - for (kind, name), (number, surface) in sorted(inventory.items(), key=lambda i: i[1][0]): - if not surface.startswith("internal: ") and surface not in sections: + if surface.startswith("internal:"): + if not surface[len("internal:") :].strip(): + report(INVALID, number, f"{kind} {name} is internal with no reason") + elif surface not in sections: report( INVALID, number, f"{kind} {name} names surface {surface!r}, which has no section" ) - if surface.startswith("internal:") and not surface[len("internal:") :].strip(): - report(INVALID, number, f"{kind} {name} is internal with no reason") seen: dict[str, int] = {} for surface, found_rows in rows.items(): @@ -164,11 +231,11 @@ def report(code: str, number: int, why: str) -> None: report(INVALID, number, f"{row_id}: unknown category {category!r}") if not rule: report(INVALID, number, f"{row_id}: empty Rule") - elif why := rule_problem(root, rule): + elif why := rule_problem(root, known, rule): report(INVALID, number, f"{row_id}: {why}") if not value: report(INVALID, number, f"{row_id}: empty Verification") - elif why := verification_problem(root, value, counts.get(category, 0) == 1): + elif why := verification_problem(root, known, value, counts.get(category, 0) == 1): report(INVALID, number, f"{row_id}: {why}") return problems From 306147cdcaf560ef74ac5ae3c912f6743e7b32de Mon Sep 17 00:00:00 2001 From: Claude Code Date: Sun, 15 Nov 2026 09:15:50 -0400 Subject: [PATCH 3/3] ci(repo): a threat-model verification is its plain repository path (#244) Re-review: a '..' segment no longer passes the allowed prefixes; the prefix is tested on the resolved path, and a test directory or a model without an inventory block is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01MFyANPkiUZYq6rvGD2sFLf --- .github/workflows/scripts/test_threat_model.py | 16 +++++++++++++++- .github/workflows/scripts/threat_model.py | 10 +++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/test_threat_model.py b/.github/workflows/scripts/test_threat_model.py index 9315b36e..4c5902ef 100644 --- a/.github/workflows/scripts/test_threat_model.py +++ b/.github/workflows/scripts/test_threat_model.py @@ -62,6 +62,7 @@ def repository(root: Path, model: str = MODEL, openapi: str = OPENAPI_TEXT) -> P (root / "docs/product/laws.md").write_text("# Laws\n\n### Law 3 — Security by design\n") (root / "docs/roadmap.md").write_text("| P3-01 | Security scenarios | conformance |\n") (root / "tests/test_api.py").write_text("") + (root / "tests/sub").mkdir() (root / "packages/one/tests/test_one.py").write_text("") (root / "README.md").write_text("") (root / MODEL_PATH).write_text(model) @@ -224,6 +225,13 @@ def test_r5_a_malformed_or_unclosed_inventory_and_a_row_outside_a_surface(self) self.problems(outside), ) + def test_a_model_without_an_inventory_block(self) -> None: + model = MODEL[: MODEL.index("```inventory")] + MODEL[MODEL.index("## Surface: api") :] + self.assertIn( + self.model_problem("THREAT_MODEL_INVALID", 1, "the model has no inventory block"), + self.problems(model), + ) + def test_r6_a_rule_naming_nothing_that_exists(self) -> None: cases = { "security.md §9": "Rule 'security.md §9' names no security.md heading", @@ -240,11 +248,17 @@ def test_r6_a_rule_naming_nothing_that_exists(self) -> None: def test_r7_a_verification_of_no_allowed_form(self) -> None: tail = "is not a test file, a scenario path, a roadmap ID, a gap or n/a" + plain = "is not written as its plain repository path" cases = { "tests/missing.py": f"Verification 'tests/missing.py' {tail}", "README.md": f"Verification 'README.md' {tail}", "tests": f"Verification 'tests' {tail}", - "../outside.py": f"Verification '../outside.py' {tail}", + "../outside.py": "Verification '../outside.py' points outside the repository", + "tests/../README.md": f"Verification 'tests/../README.md' {plain}", + "conformance/security/scenarios/../../README.md": ( + f"Verification 'conformance/security/scenarios/../../README.md' {plain}" + ), + "tests/sub": f"Verification 'tests/sub' {tail}", "tested elsewhere": f"Verification 'tested elsewhere' {tail}", "roadmap:P9-99": "Verification 'roadmap:P9-99' names no roadmap ID", "gap:": "Verification 'gap:' has no reason or no valid ID", diff --git a/.github/workflows/scripts/threat_model.py b/.github/workflows/scripts/threat_model.py index 50af4e9f..1f9c01bf 100644 --- a/.github/workflows/scripts/threat_model.py +++ b/.github/workflows/scripts/threat_model.py @@ -133,10 +133,14 @@ def verification_problem(root: Path, known: References, value: str, alone: bool) if value.startswith(("gap:", "n/a:", "roadmap:")): return f"Verification {value!r} has no reason or no valid ID" target = (root / value).resolve() - inside = root.resolve() in target.parents - if inside and TEST_FILES.match(value) and target.is_file(): + if root.resolve() not in target.parents: + return f"Verification {value!r} points outside the repository" + relative = target.relative_to(root.resolve()).as_posix() + if relative != value: + return f"Verification {value!r} is not written as its plain repository path" + if TEST_FILES.match(relative) and target.is_file(): return None - if inside and value.startswith(SCENARIOS) and target.exists(): + if relative.startswith(SCENARIOS) and target.exists(): return None return f"Verification {value!r} is not a test file, a scenario path, a roadmap ID, a gap or n/a"