From 5488dd87cabe9900bdbf8b2ba2f4369bd4009f29 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:20:57 -0700 Subject: [PATCH 1/7] feat(m8): assemble operator procedures from named fences, with lints Live sessions have stopped on composition defects, never on the code under test: a blanket empty-stderr rule against a CLI summary, a block hardcoding the previous generation's paths, quoting at a transport boundary. Each was caused by hand-copying blocks into step scripts. scripts/m8_procedure_package.py reads one TOML Composition block in a reviewed procedure document. It builds exact LF step files, their declared stdout and a SHA-256 manifest from named fences in hash-pinned frozen sources. It refuses, naming the step: - unresolvable sources, headings or fence indexes; - a lane CLI call outside lane_capture, which would have caught the g11 S4 stop; - undeclared or historical generation labels, which would have caught the H7 path gap; - unreviewed placeholders outside --template; - CRLF documents. It never runs a step. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 202 +++++++++++++++++++++++++++++ tests/test_m8_procedure_package.py | 199 ++++++++++++++++++++++++++++ 2 files changed, 401 insertions(+) create mode 100644 scripts/m8_procedure_package.py create mode 100644 tests/test_m8_procedure_package.py diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py new file mode 100644 index 00000000..c78cc3f7 --- /dev/null +++ b/scripts/m8_procedure_package.py @@ -0,0 +1,202 @@ +"""Assemble a reviewed M8 operator procedure into exact step files; never run them. + +A procedure document declares its composition in one TOML block under a heading +named ``Composition``. Steps are built from named fenced blocks, never copied by +hand:: + + procedure = "g12" + labels = {current = ["g12", "q12"], historical = ["g11", "q11"]} + + [sources.oc] # a frozen document, pinned by SHA-256 + path = "M8-N4-operator-commands.md" + sha256 = "..." + + [[steps]] + name = "c3-qualification" + stdout = ["lane-boundary-checked"] # the step's exact declared stdout lines + historical = false # may this step name historical labels? + parts = [ + {literal = "C=..."}, + {source = "oc", heading = "Setup: fixed controller and evidence checks", fences = [0, 1]}, + {source = "self", heading = "L1: qualification boundary", fences = [0]}, + ] + +``self`` is the procedure document itself; its bytes are pinned by +``docs/plans/MANIFEST.sha256``. Usage, with isolated Python from the repository +root:: + + python -I -S -B scripts/m8_procedure_package.py check DOC + python -I -S -B scripts/m8_procedure_package.py build DOC OUT [--template] + +``build`` writes ``.sh`` and ``.stdout`` files (LF only) and a +``MANIFEST.sha256`` into the absent directory ``OUT``, then prints the +manifest's own SHA-256. Every check below refuses with a message naming the +step; nothing is guessed or repaired: + +- every source pin, heading and fence index resolves exactly; +- a lane CLI call (``"${LANE[@]}"``) appears only inside ``lane_capture`` (the + #210 lane-output contract) or in the frozen held-startup form that discards its + output to ``/dev/null``; +- generation and session labels (``g12``, ``q11``...) are declared, and a step + names historical ones only if it says so; +- outside ``--template``, no ``UNREVIEWED``/``UNPREPARED`` placeholder remains. +""" + +from __future__ import annotations + +import hashlib +import re +import sys +import tomllib +from dataclasses import dataclass +from pathlib import Path + +FENCE = re.compile(r"^```(?P[a-z]*)\n(?P.*?)^```", re.MULTILINE | re.DOTALL) +HEADING = re.compile(r"^(?P#{1,6}) (?P.+?)\s*$", re.MULTILINE) +LABEL = re.compile(r"(?<![0-9A-Za-z])[gq][0-9]{1,3}(?![0-9A-Za-z])") +PLACEHOLDER = re.compile(r"\b(?:UNREVIEWED|UNPREPARED)\b") +LANE_CALL = '"${LANE[@]}"' +HELD_DISCARD = re.compile(r">\s*/dev/null\s+2>\s*/dev/null\s*&\s*$") +NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") + + +class Refused(ValueError): + """The composition is not exactly reviewable; nothing is written.""" + + +@dataclass(frozen=True) +class Step: + name: str + text: str + stdout: tuple[str, ...] + + +def section(document: str, title: str) -> str: + """The text under exactly one heading named ``title``, to the next heading at + the same or a higher level.""" + matches = [match for match in HEADING.finditer(document) if match["title"] == title] + if len(matches) != 1: + raise Refused(f"heading {title!r} occurs {len(matches)} times") + start, level = matches[0].end(), len(matches[0]["level"]) + for following in HEADING.finditer(document, start): + if len(following["level"]) <= level: + return document[start:following.start()] + return document[start:] + + +def fences(text: str) -> list[tuple[str, str]]: + return [(match["lang"], match["body"]) for match in FENCE.finditer(text)] + + +def composition(document: str) -> dict: + blocks = [body for lang, body in fences(section(document, "Composition")) if lang == "toml"] + if len(blocks) != 1: + raise Refused("the Composition section needs exactly one toml block") + try: + return tomllib.loads(blocks[0]) + except tomllib.TOMLDecodeError as error: + raise Refused(f"composition is not valid TOML: {error}") from None + + +def _lane_calls_are_captured(name: str, text: str) -> None: + logical = re.sub(r"\\\n", " ", text) + for line in logical.splitlines(): + if LANE_CALL not in line: + continue + command = line.strip() + captured = command.startswith("lane_capture ") + held = HELD_DISCARD.search(command) is not None + if not (captured or held): + raise Refused(f"step {name}: a lane CLI call is not inside lane_capture " + "(#210 lane-output contract)") + + +def _labels(name: str, text: str, labels: dict, historical: bool) -> None: + current = set(labels.get("current", ())) + allowed = current | (set(labels.get("historical", ())) if historical else set()) + stray = sorted({label for label in LABEL.findall(text) if label not in allowed}) + if stray: + raise Refused(f"step {name}: undeclared or historical labels {stray}") + + +def assemble(path: Path, template: bool = False) -> list[Step]: + # Raw bytes: text-mode reading would silently translate CRLF and hide it. + document = path.read_bytes().decode("utf-8") + if "\r" in document: + raise Refused("the procedure document is not LF-only") + spec = composition(document) + sources = {"self": document} + for key, source in spec.get("sources", {}).items(): + raw = (path.parent / source["path"]).read_bytes() + if hashlib.sha256(raw).hexdigest() != source["sha256"]: + raise Refused(f"source {key} does not match its pinned SHA-256") + if b"\r" in raw: + raise Refused(f"source {key} is not LF-only") + sources[key] = raw.decode("utf-8") + steps, names = [], set() + for entry in spec.get("steps", ()): + name = entry.get("name", "") + if not NAME.fullmatch(name) or name in names: + raise Refused(f"step name {name!r} is invalid or repeated") + names.add(name) + pieces = [] + for part in entry["parts"]: + if "literal" in part: + pieces.append(part["literal"].rstrip("\n") + "\n") + continue + if part.get("source") not in sources: + raise Refused(f"step {name}: unknown source {part.get('source')!r}") + found = fences(section(sources[part["source"]], part["heading"])) + for index in part["fences"]: + if not 0 <= index < len(found): + raise Refused(f"step {name}: {part['heading']!r} has no fence {index}") + pieces.append(found[index][1]) + text = "".join(pieces) + _lane_calls_are_captured(name, text) + _labels(name, text, spec.get("labels", {}), bool(entry.get("historical", False))) + if not template and PLACEHOLDER.search(text): + raise Refused(f"step {name}: an unreviewed placeholder remains") + stdout = tuple(entry["stdout"]) + if any("\n" in line for line in stdout): + raise Refused(f"step {name}: a declared stdout line contains a newline") + steps.append(Step(name, text, stdout)) + if not steps: + raise Refused("the composition declares no steps") + return steps + + +def build(path: Path, out: Path, template: bool = False) -> str: + steps = assemble(path, template) + out.mkdir() + lines = [] + for step in steps: + for suffix, body in ((".sh", step.text), (".stdout", "".join( + line + "\n" for line in step.stdout))): + raw = body.encode("utf-8") + (out / (step.name + suffix)).write_bytes(raw) + lines.append(f"{hashlib.sha256(raw).hexdigest()} {step.name}{suffix}\n") + manifest = "".join(lines).encode("utf-8") + (out / "MANIFEST.sha256").write_bytes(manifest) + return hashlib.sha256(manifest).hexdigest() + + +def main(argv: list[str]) -> int: + try: + if argv[1:2] == ["check"] and len(argv) == 3: + steps = assemble(Path(argv[2]), template=True) + print(f"composition-checked {len(steps)} steps") + elif argv[1:2] == ["build"] and len(argv) in (4, 5): + template = argv[4:] == ["--template"] + if len(argv) == 5 and not template: + raise Refused("unknown option") + print(build(Path(argv[2]), Path(argv[3]), template)) + else: + raise Refused("usage: check DOC | build DOC OUT [--template]") + except (Refused, OSError, KeyError, TypeError) as error: + print(f"composition refused: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py new file mode 100644 index 00000000..0a9dff76 --- /dev/null +++ b/tests/test_m8_procedure_package.py @@ -0,0 +1,199 @@ +"""The procedure packager: exact assembly, pinned sources, and refusals that name the step.""" + +from __future__ import annotations + +import hashlib +import subprocess +import sys +from pathlib import Path + +import pytest +from scripts import m8_procedure_package as package + +HANDOFFS = Path(__file__).parents[1] / "docs" / "plans" / "handoffs" +FROZEN = """# Frozen + +## Setup + +```bash +set -Eeuo pipefail +LANE=(python lane) +``` + +### S4. Qualification + +```bash +"${ROOT_STORE[@]}" maintain -- \\ + "${LANE[@]}" startup --evidence "$W/s4.json" < /dev/null +``` + +### S6a. Held + +```bash +"${ROOT_STORE[@]}" maintain -- \\ + "${LANE[@]}" startup --hold 90 \\ + < /dev/null > /dev/null 2> /dev/null & +``` +""" + + +def procedure(tmp_path: Path, steps: str, frozen: str = FROZEN, labels: str = "") -> Path: + (tmp_path / "frozen.md").write_bytes(frozen.encode()) + digest = hashlib.sha256(frozen.encode()).hexdigest() + document = f"""# Procedure + +## L1: qualification boundary + +```bash +lane_capture "$LANES/s4" "${{ROOT_STORE[@]}}" maintain -- \\ + "${{LANE[@]}}" startup --evidence "$W/s4.json" +``` + +## Composition + +```toml +procedure = "g12" +labels = {{current = ["g12", "q12"], historical = ["g11", "q11"]}} +{labels} +[sources.oc] +path = "frozen.md" +sha256 = "{digest}" +{steps} +``` +""" + path = tmp_path / "procedure.md" + path.write_bytes(document.encode()) + return path + + +QUALIFICATION = """ +[[steps]] +name = "c3-qualification" +stdout = ["lane-boundary-checked"] +parts = [ + {literal = "W=/home/m8-service/m8-maint-q12"}, + {source = "oc", heading = "Setup", fences = [0]}, + {source = "self", heading = "L1: qualification boundary", fences = [0]}, + {source = "oc", heading = "S6a. Held", fences = [0]}, +] +""" + + +def test_build_writes_exact_step_bytes_and_a_manifest(tmp_path): + out = tmp_path / "package" + digest = package.build(procedure(tmp_path, QUALIFICATION), out) + script = (out / "c3-qualification.sh").read_text(encoding="utf-8") + assert script.startswith("W=/home/m8-service/m8-maint-q12\nset -Eeuo pipefail\n") + assert 'lane_capture "$LANES/s4"' in script and "--hold 90" in script + assert (out / "c3-qualification.stdout").read_bytes() == b"lane-boundary-checked\n" + manifest = (out / "MANIFEST.sha256").read_bytes() + assert hashlib.sha256(manifest).hexdigest() == digest + for line in manifest.decode().splitlines(): + expected, name = line.split(" ") + assert hashlib.sha256((out / name).read_bytes()).hexdigest() == expected + assert b"\r" not in script.encode() + with pytest.raises(FileExistsError): + package.build(procedure(tmp_path, QUALIFICATION), out) + + +def test_frozen_unwrapped_lane_call_is_refused_like_the_g11_s4_stop(tmp_path): + steps = QUALIFICATION.replace( + '{source = "self", heading = "L1: qualification boundary", fences = [0]}', + '{source = "oc", heading = "S4. Qualification", fences = [0]}') + with pytest.raises(package.Refused, match="c3-qualification: a lane CLI call is not inside"): + package.assemble(procedure(tmp_path, steps)) + + +def test_stale_generation_paths_are_refused_like_the_h7_composition_gap(tmp_path): + stale = QUALIFICATION.replace('"W=/home/m8-service/m8-maint-q12"', + '"ACTIVE=$W/h7-g11-active.json"') + with pytest.raises(package.Refused, match=r"undeclared or historical labels \['g11'\]"): + package.assemble(procedure(tmp_path, stale)) + allowed = stale.replace('stdout = ["lane-boundary-checked"]', + 'stdout = ["lane-boundary-checked"]\nhistorical = true') + assert package.assemble(procedure(tmp_path, allowed))[0].name == "c3-qualification" + undeclared = QUALIFICATION.replace("q12", "q13") + with pytest.raises(package.Refused, match=r"\['q13'\]"): + package.assemble(procedure(tmp_path, undeclared)) + + +def test_a_changed_frozen_source_is_refused(tmp_path): + path = procedure(tmp_path, QUALIFICATION) + (tmp_path / "frozen.md").write_bytes(FROZEN.replace("90", "91").encode()) + with pytest.raises(package.Refused, match="source oc does not match its pinned SHA-256"): + package.assemble(path) + + +@pytest.mark.parametrize(("change", "message"), [ + (('heading = "Setup"', 'heading = "Absent"'), "heading 'Absent' occurs 0 times"), + (("fences = [0]}", "fences = [3]}"), "has no fence 3"), + (('source = "oc", heading = "Setup"', 'source = "other", heading = "Setup"'), + "unknown source 'other'"), + (('name = "c3-qualification"', 'name = "C3 Bad"'), "step name 'C3 Bad' is invalid"), + (('stdout = ["lane-boundary-checked"]', 'stdout = ["two\\nlines"]'), "contains a newline"), +]) +def test_unresolvable_compositions_refuse(tmp_path, change, message): + with pytest.raises(package.Refused, match=message): + package.assemble(procedure(tmp_path, QUALIFICATION.replace(*change, 1))) + + +def test_repeated_steps_and_placeholders_refuse(tmp_path): + with pytest.raises(package.Refused, match="invalid or repeated"): + package.assemble(procedure(tmp_path, QUALIFICATION * 2)) + unreviewed = QUALIFICATION.replace("m8-maint-q12", "m8-maint-q12\\nRUN_ID=UNREVIEWED") + path = procedure(tmp_path, unreviewed) + with pytest.raises(package.Refused, match="unreviewed placeholder"): + package.assemble(path) + assert package.assemble(path, template=True)[0].text.count("UNREVIEWED") == 1 + + +def test_crlf_document_and_missing_composition_refuse(tmp_path): + path = procedure(tmp_path, QUALIFICATION) + path.write_bytes(path.read_bytes().replace(b"\n", b"\r\n")) + with pytest.raises(package.Refused, match="not LF-only"): + package.assemble(path) + path.write_bytes(b"# Procedure\n\n## Other\n") + with pytest.raises(package.Refused, match="heading 'Composition' occurs 0 times"): + package.assemble(path) + + +def test_section_ends_at_the_next_heading_of_the_same_or_higher_level(): + document = "# A\n## B\nb\n### C\nc\n## D\nd\n" + assert package.section(document, "B") == "\nb\n### C\nc\n" + assert package.section(document, "C") == "\nc\n" + + +def test_cli_checks_and_builds_with_isolated_python(tmp_path): + path = procedure(tmp_path, QUALIFICATION) + script = str(Path(package.__file__)) + + def run(*args): + return subprocess.run([sys.executable, "-I", "-S", "-B", script, *args], + stdin=subprocess.DEVNULL, capture_output=True, text=True, + timeout=30, check=False) + + checked = run("check", str(path)) + assert (checked.returncode, checked.stdout) == (0, "composition-checked 1 steps\n") + built = run("build", str(path), str(tmp_path / "out")) + assert built.returncode == 0 and len(built.stdout.strip()) == 64 + refused = run("build", str(path), str(tmp_path / "out")) + assert refused.returncode == 1 and refused.stderr.startswith("composition refused:") + assert run("bogus").returncode == 1 + + +def test_the_real_frozen_lane_calls_are_classified_correctly(): + """The lint against the actual frozen bytes: OC S4 and ER R1 refuse, S6a passes.""" + operator = (HANDOFFS / "M8-N4-operator-commands.md").read_text(encoding="utf-8") + refresh = (HANDOFFS / "M8-N5-explicit-refresh.md").read_text(encoding="utf-8") + lane_output = (HANDOFFS / "M8-maintenance-lane-output.md").read_text(encoding="utf-8") + for heading, document in (("S4. Qualification", operator), ("R1: one explicit-refresh attempt", + refresh)): + block = "".join(body for _, body in package.fences(package.section(document, heading))) + with pytest.raises(package.Refused, match="not inside lane_capture"): + package._lane_calls_are_captured("frozen", block) + held = "".join(body for _, body in package.fences( + package.section(operator, "S6a. Maintenance-lock positive control"))) + package._lane_calls_are_captured("held", held) + for heading in ("L1: qualification boundary", "L2: H7 controls", "L3: explicit refresh"): + block = "".join(body for _, body in package.fences(package.section(lane_output, heading))) + package._lane_calls_are_captured(heading, block) From 97843e8d12d8012885679226956177520608b317 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:35:18 -0700 Subject: [PATCH 2/7] fix(m8): allowlist lane-boundary blocks, strict schema, CommonMark fences Review of 5488dd8 (Codex) found, all fixed here: - The line-based lane lint was bypassable (`lane_capture x true; "${LANE[@]}" ...`, a command substitution, any backgrounded call that discards its output) and also refused harmless forms. It is replaced by an allowlist with no shell parsing. Only the reviewed blocks may name the lane array: #210 L1-L3, plus OC Setup, S1 and S6a. Literals never may, and splitting the name with a backslash-newline does not hide it. - Permissive TOML shapes changed meaning: `historical = "false"` was truthy, `stdout = "ok"` became two lines, empty parts built an empty script, and a mixed literal/selection part ignored its selector. The schema is now strict. - LF was checked only on input. Escaped CR in literals or stdout is now refused. - Indented fences were invisible, four-backtick examples exposed their inner blocks, and a closer with a suffix closed the block. A CommonMark-style scanner fixes all three, and headings inside fences are ignored. - The UNPREPARED refusal sentinel was a false placeholder hit. Only assignments (`=UNREVIEWED`, `=UNPREPARED`) count now. - `check` is strict unless --template. - Selected fences must be bash. Declared stdout is bounded to 65,536 bytes. Output is built in a sibling directory and renamed into place. Labels stay a spelling check, documented as such, not proof of fresh paths. Refs #186 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 283 ++++++++++++++++++++--------- tests/test_m8_procedure_package.py | 220 ++++++++++++++++------ 2 files changed, 356 insertions(+), 147 deletions(-) diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py index c78cc3f7..71ee5fa8 100644 --- a/scripts/m8_procedure_package.py +++ b/scripts/m8_procedure_package.py @@ -4,8 +4,7 @@ named ``Composition``. Steps are built from named fenced blocks, never copied by hand:: - procedure = "g12" - labels = {current = ["g12", "q12"], historical = ["g11", "q11"]} + labels = {current = ["g11", "q12"], historical = ["g10", "q11"]} [sources.oc] # a frozen document, pinned by SHA-256 path = "M8-N4-operator-commands.md" @@ -18,52 +17,77 @@ parts = [ {literal = "C=..."}, {source = "oc", heading = "Setup: fixed controller and evidence checks", fences = [0, 1]}, - {source = "self", heading = "L1: qualification boundary", fences = [0]}, + {source = "lane", heading = "L1: qualification boundary", fences = [0]}, ] -``self`` is the procedure document itself; its bytes are pinned by -``docs/plans/MANIFEST.sha256``. Usage, with isolated Python from the repository -root:: +``self`` is the procedure document itself, pinned by ``docs/plans/MANIFEST.sha256``. +Usage, with isolated Python from the repository root:: - python -I -S -B scripts/m8_procedure_package.py check DOC + python -I -S -B scripts/m8_procedure_package.py check DOC [--template] python -I -S -B scripts/m8_procedure_package.py build DOC OUT [--template] -``build`` writes ``<step>.sh`` and ``<step>.stdout`` files (LF only) and a -``MANIFEST.sha256`` into the absent directory ``OUT``, then prints the -manifest's own SHA-256. Every check below refuses with a message naming the -step; nothing is guessed or repaired: - -- every source pin, heading and fence index resolves exactly; -- a lane CLI call (``"${LANE[@]}"``) appears only inside ``lane_capture`` (the - #210 lane-output contract) or in the frozen held-startup form that discards its - output to ``/dev/null``; -- generation and session labels (``g12``, ``q11``...) are declared, and a step - names historical ones only if it says so; -- outside ``--template``, no ``UNREVIEWED``/``UNPREPARED`` placeholder remains. +``build`` writes ``<step>.sh`` and ``<step>.stdout`` (LF only) and ``MANIFEST.sha256`` +(``sha256sum --check`` format) into the absent directory ``OUT``, atomically by +renaming a completed sibling, then prints the manifest's own SHA-256. It refuses, +naming the step, and never guesses or repairs: + +- **schema:** unknown keys, wrong types, empty parts or selections, a part that + is not exactly a literal or exactly a fence selection; +- **sources:** a changed pin, a CRLF source, an absent or repeated heading, a + missing fence, a fence that is not ``bash``; +- **lane boundary (#210):** text naming the lane CLI array (``LANE``) unless it + comes from an allowlisted, reviewed lane-boundary block; literals never may; +- **labels:** an undeclared generation/session label, or a historical one in a + step not marked historical. This is a spelling check, not proof of fresh paths; +- **placeholders:** an ``=UNREVIEWED``/``=UNPREPARED`` assignment, unless ``--template``; +- **bytes:** any CR in a built file, a declared stdout over 65,536 bytes. """ from __future__ import annotations import hashlib +import os import re import sys import tomllib from dataclasses import dataclass from pathlib import Path -FENCE = re.compile(r"^```(?P<lang>[a-z]*)\n(?P<body>.*?)^```", re.MULTILINE | re.DOTALL) -HEADING = re.compile(r"^(?P<level>#{1,6}) (?P<title>.+?)\s*$", re.MULTILINE) +HEADING = re.compile(r"^(?P<level>#{1,6}) (?P<title>.+?)[ \t]*$") +OPEN_FENCE = re.compile(r"^(?P<indent> {0,3})(?P<ticks>`{3,})(?P<lang>[A-Za-z]*)[ \t]*$") LABEL = re.compile(r"(?<![0-9A-Za-z])[gq][0-9]{1,3}(?![0-9A-Za-z])") -PLACEHOLDER = re.compile(r"\b(?:UNREVIEWED|UNPREPARED)\b") -LANE_CALL = '"${LANE[@]}"' -HELD_DISCARD = re.compile(r">\s*/dev/null\s+2>\s*/dev/null\s*&\s*$") +PLACEHOLDER = re.compile(r"=(?:UNREVIEWED|UNPREPARED)\b") +LANE_NAME = re.compile(r"\bLANE\b") NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") +MAX_STDOUT = 65_536 +# The only reviewed blocks that may name the lane CLI. Startup boundaries are +# wrapped by lane_capture (#210 L1-L3); OC Setup defines the array, S1 prepare +# prints a declared record, and S6a discards its held lane's output then checks +# `wait` and its evidence. +LANE_BLOCKS = { + ("M8-maintenance-lane-output.md", "L1: qualification boundary"), + ("M8-maintenance-lane-output.md", "L2: H7 controls"), + ("M8-maintenance-lane-output.md", "L3: explicit refresh"), + ("M8-N4-operator-commands.md", "Setup: fixed controller and evidence checks"), + ("M8-N4-operator-commands.md", "S1. Prepare"), + ("M8-N4-operator-commands.md", "S6a. Maintenance-lock positive control"), +} +TOP_KEYS = {"labels", "sources", "steps"} +STEP_KEYS = {"name", "stdout", "historical", "parts"} class Refused(ValueError): """The composition is not exactly reviewable; nothing is written.""" +@dataclass(frozen=True) +class Fence: + lang: str + body: str + start: int + end: int + + @dataclass(frozen=True) class Step: name: str @@ -71,21 +95,69 @@ class Step: stdout: tuple[str, ...] +def fence_blocks(document: str) -> list[Fence]: + """CommonMark-style backtick fences: up to three spaces of indent, a closing + run at least as long as the opening and nothing after it.""" + lines = document.splitlines(keepends=True) + starts = [0] + for line in lines: + starts.append(starts[-1] + len(line)) + blocks, index = [], 0 + while index < len(lines): + opening = OPEN_FENCE.match(lines[index].rstrip("\n")) + if opening is None: + index += 1 + continue + indent, ticks = len(opening["indent"]), len(opening["ticks"]) + closer = re.compile(rf"^ {{0,3}}`{{{ticks},}}[ \t]*$") + body, cursor = [], index + 1 + while cursor < len(lines) and not closer.match(lines[cursor].rstrip("\n")): + line = lines[cursor] + body.append(line[min(indent, len(line) - len(line.lstrip(" "))):]) + cursor += 1 + if cursor == len(lines): + raise Refused("a fenced block is never closed") + blocks.append(Fence(opening["lang"], "".join(body), starts[index], starts[cursor + 1])) + index = cursor + 1 + return blocks + + +@dataclass(frozen=True) +class Heading: + start: int # the heading line's first character + end: int # just after the heading line + level: int + title: str + + +def headings(document: str) -> list[Heading]: + """Markdown headings, never a ``# comment`` line inside a fenced block.""" + spans = [(block.start, block.end) for block in fence_blocks(document)] + found, offset = [], 0 + for line in document.splitlines(keepends=True): + match = HEADING.match(line.rstrip("\n")) + if match and not any(start <= offset < end for start, end in spans): + found.append(Heading(offset, offset + len(line), len(match["level"]), + match["title"])) + offset += len(line) + return found + + def section(document: str, title: str) -> str: """The text under exactly one heading named ``title``, to the next heading at the same or a higher level.""" - matches = [match for match in HEADING.finditer(document) if match["title"] == title] + found = headings(document) + matches = [heading for heading in found if heading.title == title] if len(matches) != 1: raise Refused(f"heading {title!r} occurs {len(matches)} times") - start, level = matches[0].end(), len(matches[0]["level"]) - for following in HEADING.finditer(document, start): - if len(following["level"]) <= level: - return document[start:following.start()] - return document[start:] + chosen = matches[0] + following = (heading.start for heading in found + if heading.start > chosen.start and heading.level <= chosen.level) + return document[chosen.end:next(following, len(document))] def fences(text: str) -> list[tuple[str, str]]: - return [(match["lang"], match["body"]) for match in FENCE.finditer(text)] + return [(block.lang, block.body) for block in fence_blocks(text)] def composition(document: str) -> dict: @@ -93,30 +165,44 @@ def composition(document: str) -> dict: if len(blocks) != 1: raise Refused("the Composition section needs exactly one toml block") try: - return tomllib.loads(blocks[0]) + spec = tomllib.loads(blocks[0]) except tomllib.TOMLDecodeError as error: raise Refused(f"composition is not valid TOML: {error}") from None - - -def _lane_calls_are_captured(name: str, text: str) -> None: - logical = re.sub(r"\\\n", " ", text) - for line in logical.splitlines(): - if LANE_CALL not in line: - continue - command = line.strip() - captured = command.startswith("lane_capture ") - held = HELD_DISCARD.search(command) is not None - if not (captured or held): - raise Refused(f"step {name}: a lane CLI call is not inside lane_capture " - "(#210 lane-output contract)") - - -def _labels(name: str, text: str, labels: dict, historical: bool) -> None: - current = set(labels.get("current", ())) - allowed = current | (set(labels.get("historical", ())) if historical else set()) - stray = sorted({label for label in LABEL.findall(text) if label not in allowed}) - if stray: - raise Refused(f"step {name}: undeclared or historical labels {stray}") + if not set(spec) <= TOP_KEYS: + raise Refused(f"unknown composition keys {sorted(set(spec) - TOP_KEYS)}") + return spec + + +def _strings(value: object, what: str) -> list[str]: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise Refused(f"{what} must be a list of strings") + return value + + +def _part(name: str, part: object, sources: dict, paths: dict) -> tuple[str, str | None]: + """(text, allowlist key or None) for one part, after strict shape checks.""" + if not isinstance(part, dict): + raise Refused(f"step {name}: a part must be a table") + if set(part) == {"literal"} and isinstance(part["literal"], str): + return part["literal"].rstrip("\n") + "\n", None + if set(part) != {"source", "heading", "fences"}: + raise Refused(f"step {name}: a part is exactly a literal or a fence selection") + source, heading, selected = part["source"], part["heading"], part["fences"] + if source not in sources or not isinstance(heading, str): + raise Refused(f"step {name}: unknown source {source!r}") + if not isinstance(selected, list) or not selected or not all( + type(index) is int for index in selected): + raise Refused(f"step {name}: fences must be a non-empty list of integers") + found = fences(section(sources[source], heading)) + text = [] + for index in selected: + if not 0 <= index < len(found): + raise Refused(f"step {name}: {heading!r} has no fence {index}") + lang, body = found[index] + if lang != "bash": + raise Refused(f"step {name}: {heading!r} fence {index} is not bash") + text.append(body) + return "".join(text), (paths[source], heading) def assemble(path: Path, template: bool = False) -> list[Step]: @@ -125,40 +211,56 @@ def assemble(path: Path, template: bool = False) -> list[Step]: if "\r" in document: raise Refused("the procedure document is not LF-only") spec = composition(document) - sources = {"self": document} + sources, paths = {"self": document}, {"self": path.name} for key, source in spec.get("sources", {}).items(): + if not isinstance(source, dict) or set(source) != {"path", "sha256"}: + raise Refused(f"source {key} needs exactly path and sha256") raw = (path.parent / source["path"]).read_bytes() if hashlib.sha256(raw).hexdigest() != source["sha256"]: raise Refused(f"source {key} does not match its pinned SHA-256") if b"\r" in raw: raise Refused(f"source {key} is not LF-only") - sources[key] = raw.decode("utf-8") + sources[key], paths[key] = raw.decode("utf-8"), Path(source["path"]).name + labels = spec.get("labels", {}) + if not isinstance(labels, dict) or not set(labels) <= {"current", "historical"}: + raise Refused("labels has only current and historical lists") + current = set(_strings(labels.get("current", []), "labels.current")) + historical_labels = set(_strings(labels.get("historical", []), "labels.historical")) steps, names = [], set() - for entry in spec.get("steps", ()): - name = entry.get("name", "") - if not NAME.fullmatch(name) or name in names: + for entry in spec.get("steps", []): + if not isinstance(entry, dict) or not set(entry) <= STEP_KEYS or not { + "name", "stdout", "parts"} <= set(entry): + raise Refused("a step has exactly name, stdout, parts and optional historical") + name = entry["name"] + if not isinstance(name, str) or not NAME.fullmatch(name) or name in names: raise Refused(f"step name {name!r} is invalid or repeated") names.add(name) + historical = entry.get("historical", False) + if type(historical) is not bool: + raise Refused(f"step {name}: historical must be a boolean") + parts = entry["parts"] + if not isinstance(parts, list) or not parts: + raise Refused(f"step {name}: parts must be a non-empty list") pieces = [] - for part in entry["parts"]: - if "literal" in part: - pieces.append(part["literal"].rstrip("\n") + "\n") - continue - if part.get("source") not in sources: - raise Refused(f"step {name}: unknown source {part.get('source')!r}") - found = fences(section(sources[part["source"]], part["heading"])) - for index in part["fences"]: - if not 0 <= index < len(found): - raise Refused(f"step {name}: {part['heading']!r} has no fence {index}") - pieces.append(found[index][1]) + for part in parts: + text, origin = _part(name, part, sources, paths) + if LANE_NAME.search(text.replace("\\\n", "")) and origin not in LANE_BLOCKS: + raise Refused(f"step {name}: only a reviewed lane-boundary block may name " + "the lane CLI (#210 lane-output contract)") + pieces.append(text) text = "".join(pieces) - _lane_calls_are_captured(name, text) - _labels(name, text, spec.get("labels", {}), bool(entry.get("historical", False))) + allowed = current | (historical_labels if historical else set()) + stray = sorted({label for label in LABEL.findall(text) if label not in allowed}) + if stray: + raise Refused(f"step {name}: undeclared or historical labels {stray}") if not template and PLACEHOLDER.search(text): raise Refused(f"step {name}: an unreviewed placeholder remains") - stdout = tuple(entry["stdout"]) - if any("\n" in line for line in stdout): - raise Refused(f"step {name}: a declared stdout line contains a newline") + stdout = tuple(_strings(entry["stdout"], f"step {name}: stdout")) + rendered = "".join(line + "\n" for line in stdout) + if any("\n" in line or "\r" in line for line in stdout) or "\r" in text: + raise Refused(f"step {name}: a built file would not be LF-only") + if len(rendered.encode("utf-8")) > MAX_STDOUT: + raise Refused(f"step {name}: declared stdout exceeds {MAX_STDOUT} bytes") steps.append(Step(name, text, stdout)) if not steps: raise Refused("the composition declares no steps") @@ -167,32 +269,37 @@ def assemble(path: Path, template: bool = False) -> list[Step]: def build(path: Path, out: Path, template: bool = False) -> str: steps = assemble(path, template) - out.mkdir() + if os.path.lexists(out): + raise Refused("the output directory already exists") + partial = out.with_name(f".{out.name}.partial-{os.getpid()}") + partial.mkdir() lines = [] for step in steps: - for suffix, body in ((".sh", step.text), (".stdout", "".join( - line + "\n" for line in step.stdout))): + for suffix, body in ((".sh", step.text), + (".stdout", "".join(line + "\n" for line in step.stdout))): raw = body.encode("utf-8") - (out / (step.name + suffix)).write_bytes(raw) + (partial / (step.name + suffix)).write_bytes(raw) lines.append(f"{hashlib.sha256(raw).hexdigest()} {step.name}{suffix}\n") manifest = "".join(lines).encode("utf-8") - (out / "MANIFEST.sha256").write_bytes(manifest) + (partial / "MANIFEST.sha256").write_bytes(manifest) + os.rename(partial, out) # refuses if OUT appeared meanwhile; the partial stays visible return hashlib.sha256(manifest).hexdigest() def main(argv: list[str]) -> int: try: - if argv[1:2] == ["check"] and len(argv) == 3: - steps = assemble(Path(argv[2]), template=True) + arguments = argv[1:] + template = arguments[-1:] == ["--template"] + if template: + arguments = arguments[:-1] + if arguments[:1] == ["check"] and len(arguments) == 2: + steps = assemble(Path(arguments[1]), template) print(f"composition-checked {len(steps)} steps") - elif argv[1:2] == ["build"] and len(argv) in (4, 5): - template = argv[4:] == ["--template"] - if len(argv) == 5 and not template: - raise Refused("unknown option") - print(build(Path(argv[2]), Path(argv[3]), template)) + elif arguments[:1] == ["build"] and len(arguments) == 3: + print(build(Path(arguments[1]), Path(arguments[2]), template)) else: - raise Refused("usage: check DOC | build DOC OUT [--template]") - except (Refused, OSError, KeyError, TypeError) as error: + raise Refused("usage: check DOC [--template] | build DOC OUT [--template]") + except (Refused, OSError, UnicodeDecodeError) as error: print(f"composition refused: {error}", file=sys.stderr) return 1 return 0 diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py index 0a9dff76..fe7887f1 100644 --- a/tests/test_m8_procedure_package.py +++ b/tests/test_m8_procedure_package.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import shutil import subprocess import sys from pathlib import Path @@ -11,6 +12,7 @@ from scripts import m8_procedure_package as package HANDOFFS = Path(__file__).parents[1] / "docs" / "plans" / "handoffs" +REAL_LANE_BLOCKS = package.LANE_BLOCKS FROZEN = """# Frozen ## Setup @@ -33,28 +35,56 @@ "${ROOT_STORE[@]}" maintain -- \\ "${LANE[@]}" startup --hold 90 \\ < /dev/null > /dev/null 2> /dev/null & +wait "$H" +``` + +### Notes + +```text +not a command ``` """ +SELF_BLOCKS = """## L1: qualification boundary +```bash +lane_capture "$LANES/s4" "${ROOT_STORE[@]}" maintain -- \\ + "${LANE[@]}" startup --evidence "$W/s4.json" +``` -def procedure(tmp_path: Path, steps: str, frozen: str = FROZEN, labels: str = "") -> Path: - (tmp_path / "frozen.md").write_bytes(frozen.encode()) - digest = hashlib.sha256(frozen.encode()).hexdigest() - document = f"""# Procedure +## Unreviewed + +```bash +lane_capture "$OUT" true; "${LANE[@]}" startup +``` -## L1: qualification boundary +## Split ```bash -lane_capture "$LANES/s4" "${{ROOT_STORE[@]}}" maintain -- \\ - "${{LANE[@]}}" startup --evidence "$W/s4.json" +"${LA\\ +NE[@]}" startup ``` +""" + +@pytest.fixture(autouse=True) +def fixture_allowlist(monkeypatch): + monkeypatch.setattr(package, "LANE_BLOCKS", { + ("procedure.md", "L1: qualification boundary"), + ("frozen.md", "Setup"), ("frozen.md", "S6a. Held"), + }) + + +def procedure(tmp_path: Path, steps: str, frozen: str = FROZEN, extra: str = "") -> Path: + (tmp_path / "frozen.md").write_bytes(frozen.encode()) + digest = hashlib.sha256(frozen.encode()).hexdigest() + document = f"""# Procedure + +{SELF_BLOCKS} ## Composition ```toml -procedure = "g12" -labels = {{current = ["g12", "q12"], historical = ["g11", "q11"]}} -{labels} +labels = {{current = ["g11", "q12"], historical = ["g10", "q11"]}} +{extra} [sources.oc] path = "frozen.md" sha256 = "{digest}" @@ -79,72 +109,113 @@ def procedure(tmp_path: Path, steps: str, frozen: str = FROZEN, labels: str = "" """ -def test_build_writes_exact_step_bytes_and_a_manifest(tmp_path): +def test_build_writes_exact_step_bytes_and_a_manifest_atomically(tmp_path): out = tmp_path / "package" digest = package.build(procedure(tmp_path, QUALIFICATION), out) - script = (out / "c3-qualification.sh").read_text(encoding="utf-8") + script = (out / "c3-qualification.sh").read_bytes().decode() assert script.startswith("W=/home/m8-service/m8-maint-q12\nset -Eeuo pipefail\n") - assert 'lane_capture "$LANES/s4"' in script and "--hold 90" in script + assert 'lane_capture "$LANES/s4"' in script and 'wait "$H"' in script assert (out / "c3-qualification.stdout").read_bytes() == b"lane-boundary-checked\n" manifest = (out / "MANIFEST.sha256").read_bytes() assert hashlib.sha256(manifest).hexdigest() == digest for line in manifest.decode().splitlines(): expected, name = line.split(" ") assert hashlib.sha256((out / name).read_bytes()).hexdigest() == expected - assert b"\r" not in script.encode() - with pytest.raises(FileExistsError): + assert "\r" not in script and not list(tmp_path.glob(".package.partial-*")) + with pytest.raises(package.Refused, match="already exists"): package.build(procedure(tmp_path, QUALIFICATION), out) -def test_frozen_unwrapped_lane_call_is_refused_like_the_g11_s4_stop(tmp_path): - steps = QUALIFICATION.replace( - '{source = "self", heading = "L1: qualification boundary", fences = [0]}', - '{source = "oc", heading = "S4. Qualification", fences = [0]}') - with pytest.raises(package.Refused, match="c3-qualification: a lane CLI call is not inside"): - package.assemble(procedure(tmp_path, steps)) +@pytest.mark.parametrize("change", [ + # The g11 S4 stop: the frozen, unwrapped lane call. + ('{source = "self", heading = "L1: qualification boundary", fences = [0]}', + '{source = "oc", heading = "S4. Qualification", fences = [0]}'), + # A capture on the line does not make another call on it captured. + ('{source = "self", heading = "L1: qualification boundary", fences = [0]}', + '{source = "self", heading = "Unreviewed", fences = [0]}'), + # Splitting the name with a line continuation does not hide it. + ('{source = "self", heading = "L1: qualification boundary", fences = [0]}', + '{source = "self", heading = "Split", fences = [0]}'), + # Literals never name the lane. + ('{literal = "W=/home/m8-service/m8-maint-q12"}', + "{literal = 'OTHER=(\"${LANE[@]}\")'}"), +]) +def test_only_reviewed_lane_boundary_blocks_may_name_the_lane(tmp_path, change): + with pytest.raises(package.Refused, match="only a reviewed lane-boundary block"): + package.assemble(procedure(tmp_path, QUALIFICATION.replace(*change))) def test_stale_generation_paths_are_refused_like_the_h7_composition_gap(tmp_path): stale = QUALIFICATION.replace('"W=/home/m8-service/m8-maint-q12"', - '"ACTIVE=$W/h7-g11-active.json"') - with pytest.raises(package.Refused, match=r"undeclared or historical labels \['g11'\]"): + '"ACTIVE=$W/h7-q11-active.json"') + with pytest.raises(package.Refused, match=r"undeclared or historical labels \['q11'\]"): package.assemble(procedure(tmp_path, stale)) allowed = stale.replace('stdout = ["lane-boundary-checked"]', 'stdout = ["lane-boundary-checked"]\nhistorical = true') assert package.assemble(procedure(tmp_path, allowed))[0].name == "c3-qualification" - undeclared = QUALIFICATION.replace("q12", "q13") with pytest.raises(package.Refused, match=r"\['q13'\]"): - package.assemble(procedure(tmp_path, undeclared)) + package.assemble(procedure(tmp_path, QUALIFICATION.replace("q12", "q13"))) -def test_a_changed_frozen_source_is_refused(tmp_path): +def test_a_changed_or_crlf_frozen_source_is_refused(tmp_path): path = procedure(tmp_path, QUALIFICATION) (tmp_path / "frozen.md").write_bytes(FROZEN.replace("90", "91").encode()) with pytest.raises(package.Refused, match="source oc does not match its pinned SHA-256"): package.assemble(path) + crlf = FROZEN.replace("\n", "\r\n") + with pytest.raises(package.Refused, match="source oc is not LF-only"): + package.assemble(procedure(tmp_path, QUALIFICATION, frozen=crlf)) @pytest.mark.parametrize(("change", "message"), [ (('heading = "Setup"', 'heading = "Absent"'), "heading 'Absent' occurs 0 times"), - (("fences = [0]}", "fences = [3]}"), "has no fence 3"), + (('"Setup", fences = [0]', '"Setup", fences = [9]'), "has no fence 9"), + (('"Setup", fences = [0]', '"Setup", fences = []'), "non-empty list of integers"), + (('"Setup", fences = [0]', '"Setup", fences = ["0"]'), "non-empty list of integers"), + (('heading = "Setup"', 'heading = "Notes"'), "is not bash"), (('source = "oc", heading = "Setup"', 'source = "other", heading = "Setup"'), "unknown source 'other'"), (('name = "c3-qualification"', 'name = "C3 Bad"'), "step name 'C3 Bad' is invalid"), - (('stdout = ["lane-boundary-checked"]', 'stdout = ["two\\nlines"]'), "contains a newline"), + (('stdout = ["lane-boundary-checked"]', 'stdout = "ok"'), "stdout must be a list"), + (('stdout = ["lane-boundary-checked"]', 'stdout = ["ok\\r"]'), "not be LF-only"), + (('stdout = ["lane-boundary-checked"]', 'stdout = ["two\\nlines"]'), "not be LF-only"), + (('stdout = ["lane-boundary-checked"]', 'stdout = ["x"]\nhistorical = "false"'), + "historical must be a boolean"), + (('stdout = ["lane-boundary-checked"]', 'stdout = ["x"]\nextra = 1'), "a step has exactly"), + (('{literal = "W=/home/m8-service/m8-maint-q12"}', + '{literal = "W=x", source = "oc", heading = "Setup", fences = [0]}'), + "exactly a literal or a fence selection"), + (('{literal = "W=/home/m8-service/m8-maint-q12"}', '{literal = "true\\r\\n"}'), + "not be LF-only"), ]) -def test_unresolvable_compositions_refuse(tmp_path, change, message): +def test_unresolvable_or_ambiguous_compositions_refuse(tmp_path, change, message): with pytest.raises(package.Refused, match=message): package.assemble(procedure(tmp_path, QUALIFICATION.replace(*change, 1))) -def test_repeated_steps_and_placeholders_refuse(tmp_path): +def test_empty_parts_unknown_keys_and_oversized_stdout_refuse(tmp_path): + empty = QUALIFICATION.split("parts = [", 1)[0] + "parts = []\n" + with pytest.raises(package.Refused, match="parts must be a non-empty list"): + package.assemble(procedure(tmp_path, empty)) + with pytest.raises(package.Refused, match="unknown composition keys"): + package.assemble(procedure(tmp_path, QUALIFICATION, extra='procedure = "g12"')) + big = QUALIFICATION.replace('stdout = ["lane-boundary-checked"]', + f'stdout = ["{"x" * 65_536}"]') + with pytest.raises(package.Refused, match="exceeds 65536 bytes"): + package.assemble(procedure(tmp_path, big)) + + +def test_repeated_steps_and_assigned_placeholders_refuse_sentinels_do_not(tmp_path): with pytest.raises(package.Refused, match="invalid or repeated"): package.assemble(procedure(tmp_path, QUALIFICATION * 2)) unreviewed = QUALIFICATION.replace("m8-maint-q12", "m8-maint-q12\\nRUN_ID=UNREVIEWED") path = procedure(tmp_path, unreviewed) with pytest.raises(package.Refused, match="unreviewed placeholder"): package.assemble(path) - assert package.assemble(path, template=True)[0].text.count("UNREVIEWED") == 1 + assert package.assemble(path, template=True)[0].text.count("=UNREVIEWED") == 1 + sentinel = QUALIFICATION.replace( + "m8-maint-q12", "m8-maint-q12\\ntest \\\"$B\\\" != /h/UNPREPARED.bundle") + assert "UNPREPARED.bundle" in package.assemble(procedure(tmp_path, sentinel))[0].text def test_crlf_document_and_missing_composition_refuse(tmp_path): @@ -159,12 +230,30 @@ def test_crlf_document_and_missing_composition_refuse(tmp_path): def test_section_ends_at_the_next_heading_of_the_same_or_higher_level(): document = "# A\n## B\nb\n### C\nc\n## D\nd\n" - assert package.section(document, "B") == "\nb\n### C\nc\n" - assert package.section(document, "C") == "\nc\n" - - -def test_cli_checks_and_builds_with_isolated_python(tmp_path): - path = procedure(tmp_path, QUALIFICATION) + assert package.section(document, "B") == "b\n### C\nc\n" + assert package.section(document, "C") == "c\n" + + +def test_fences_follow_commonmark_indent_length_and_exact_closers(): + document = ("## B\n\n```bash\n# D\nx=1\n## C\n```\n\n" + " ```bash\n indented\n ```\n\n" + "````text\n```bash\ninner example\n```\n````\n\n" + "```bash\nnot closed by\n```suffix\nstill inside\n```\n\n## C\nc\n") + assert package.fences(package.section(document, "B")) == [ + ("bash", "# D\nx=1\n## C\n"), ("bash", "indented\n"), + ("text", "```bash\ninner example\n```\n"), + ("bash", "not closed by\n```suffix\nstill inside\n"), + ] + assert package.section(document, "C") == "c\n" + with pytest.raises(package.Refused, match="'D' occurs 0 times"): + package.section(document, "D") + with pytest.raises(package.Refused, match="never closed"): + package.fences("```bash\nopen\n") + + +def test_cli_is_strict_unless_template_and_runs_isolated(tmp_path): + path = procedure(tmp_path, QUALIFICATION.replace( + "m8-maint-q12", "m8-maint-q12\\nRUN_ID=UNREVIEWED")) script = str(Path(package.__file__)) def run(*args): @@ -172,28 +261,41 @@ def run(*args): stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=30, check=False) - checked = run("check", str(path)) - assert (checked.returncode, checked.stdout) == (0, "composition-checked 1 steps\n") - built = run("build", str(path), str(tmp_path / "out")) - assert built.returncode == 0 and len(built.stdout.strip()) == 64 - refused = run("build", str(path), str(tmp_path / "out")) - assert refused.returncode == 1 and refused.stderr.startswith("composition refused:") + # The isolated CLI uses the real allowlist; this fixture's L1 is not on it. + refused = run("check", str(path), "--template") + assert refused.returncode == 1 and "lane-boundary block" in refused.stderr assert run("bogus").returncode == 1 -def test_the_real_frozen_lane_calls_are_classified_correctly(): - """The lint against the actual frozen bytes: OC S4 and ER R1 refuse, S6a passes.""" - operator = (HANDOFFS / "M8-N4-operator-commands.md").read_text(encoding="utf-8") - refresh = (HANDOFFS / "M8-N5-explicit-refresh.md").read_text(encoding="utf-8") - lane_output = (HANDOFFS / "M8-maintenance-lane-output.md").read_text(encoding="utf-8") - for heading, document in (("S4. Qualification", operator), ("R1: one explicit-refresh attempt", - refresh)): - block = "".join(body for _, body in package.fences(package.section(document, heading))) - with pytest.raises(package.Refused, match="not inside lane_capture"): - package._lane_calls_are_captured("frozen", block) - held = "".join(body for _, body in package.fences( - package.section(operator, "S6a. Maintenance-lock positive control"))) - package._lane_calls_are_captured("held", held) - for heading in ("L1: qualification boundary", "L2: H7 controls", "L3: explicit refresh"): - block = "".join(body for _, body in package.fences(package.section(lane_output, heading))) - package._lane_calls_are_captured(heading, block) +def test_real_frozen_and_reviewed_blocks_are_classified(tmp_path, monkeypatch): + """The allowlist against actual repository bytes: L1-L3 and frozen S6a pass; + frozen OC S4 and ER R1, which print the lane summary unwrapped, refuse.""" + monkeypatch.setattr(package, "LANE_BLOCKS", REAL_LANE_BLOCKS) + names = ("M8-N4-operator-commands.md", "M8-N5-explicit-refresh.md", + "M8-maintenance-lane-output.md") + for name in names: + shutil.copyfile(HANDOFFS / name, tmp_path / name) + sources = "\n".join( + f'[sources.s{index}]\npath = "{name}"\nsha256 = "' + f'{hashlib.sha256((HANDOFFS / name).read_bytes()).hexdigest()}"' + for index, name in enumerate(names)) + + def step(source: str, heading: str) -> Path: + document = (f"# P\n\n## Composition\n\n```toml\nlabels = {{current = []}}\n{sources}\n" + f'[[steps]]\nname = "s"\nstdout = []\nparts = [{{source = "{source}", ' + f'heading = "{heading}", fences = [0]}}]\n```\n') + path = tmp_path / "procedure.md" + path.write_bytes(document.encode()) + return path + + for source, heading in (("s0", "S4. Qualification"), + ("s1", "R1: one explicit-refresh attempt")): + with pytest.raises(package.Refused, match="lane-boundary block"): + package.assemble(step(source, heading), template=True) + for source, heading in (("s2", "L1: qualification boundary"), ("s2", "L2: H7 controls"), + ("s2", "L3: explicit refresh"), + ("s0", "S6a. Maintenance-lock positive control")): + try: + package.assemble(step(source, heading), template=True) + except package.Refused as error: + assert "lane-boundary" not in str(error), heading From 148aacef14ef7c5ab79ac46e1cd0c3cab3813fa3 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:25:22 -0700 Subject: [PATCH 3/7] fix(m8): refuse quoted placeholders, bad sources, empty literals; claim OUT Connector review of 97843e8 found these, all fixed: - Quoted placeholder assignments (RUN_ID="UNREVIEWED") passed the check. - Malformed sources (sources = [], a non-string path, a source named self) raised a traceback instead of a refusal. - An empty literal part was silently accepted. - On POSIX the final rename could replace an empty OUT that appeared meanwhile. OUT is now claimed with an exclusive mkdir, files are created with x mode, and MANIFEST.sha256 is written last, so a package without a manifest is incomplete. Refs #186 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 37 +++++++++++++++++++----------- tests/test_m8_procedure_package.py | 28 +++++++++++++++++++++- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py index 71ee5fa8..575721a0 100644 --- a/scripts/m8_procedure_package.py +++ b/scripts/m8_procedure_package.py @@ -27,8 +27,9 @@ python -I -S -B scripts/m8_procedure_package.py build DOC OUT [--template] ``build`` writes ``<step>.sh`` and ``<step>.stdout`` (LF only) and ``MANIFEST.sha256`` -(``sha256sum --check`` format) into the absent directory ``OUT``, atomically by -renaming a completed sibling, then prints the manifest's own SHA-256. It refuses, +(``sha256sum --check`` format) into ``OUT``, which it claims with an exclusive +``mkdir``. The manifest is written last, so a package without one is incomplete. It +prints the manifest's own SHA-256 and refuses, naming the step, and never guesses or repairs: - **schema:** unknown keys, wrong types, empty parts or selections, a part that @@ -46,7 +47,6 @@ from __future__ import annotations import hashlib -import os import re import sys import tomllib @@ -56,7 +56,7 @@ HEADING = re.compile(r"^(?P<level>#{1,6}) (?P<title>.+?)[ \t]*$") OPEN_FENCE = re.compile(r"^(?P<indent> {0,3})(?P<ticks>`{3,})(?P<lang>[A-Za-z]*)[ \t]*$") LABEL = re.compile(r"(?<![0-9A-Za-z])[gq][0-9]{1,3}(?![0-9A-Za-z])") -PLACEHOLDER = re.compile(r"=(?:UNREVIEWED|UNPREPARED)\b") +PLACEHOLDER = re.compile(r"""=["']?(?:UNREVIEWED|UNPREPARED)\b""") LANE_NAME = re.compile(r"\bLANE\b") NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") MAX_STDOUT = 65_536 @@ -184,6 +184,8 @@ def _part(name: str, part: object, sources: dict, paths: dict) -> tuple[str, str if not isinstance(part, dict): raise Refused(f"step {name}: a part must be a table") if set(part) == {"literal"} and isinstance(part["literal"], str): + if not part["literal"].strip(): + raise Refused(f"step {name}: a literal part is empty") return part["literal"].rstrip("\n") + "\n", None if set(part) != {"source", "heading", "fences"}: raise Refused(f"step {name}: a part is exactly a literal or a fence selection") @@ -212,9 +214,14 @@ def assemble(path: Path, template: bool = False) -> list[Step]: raise Refused("the procedure document is not LF-only") spec = composition(document) sources, paths = {"self": document}, {"self": path.name} - for key, source in spec.get("sources", {}).items(): - if not isinstance(source, dict) or set(source) != {"path", "sha256"}: - raise Refused(f"source {key} needs exactly path and sha256") + declared = spec.get("sources", {}) + if not isinstance(declared, dict): + raise Refused("sources must be a table") + for key, source in declared.items(): + if (not isinstance(source, dict) or set(source) != {"path", "sha256"} + or not all(isinstance(source[field], str) for field in ("path", "sha256")) + or key == "self"): + raise Refused(f"source {key} needs exactly a string path and sha256") raw = (path.parent / source["path"]).read_bytes() if hashlib.sha256(raw).hexdigest() != source["sha256"]: raise Refused(f"source {key} does not match its pinned SHA-256") @@ -269,20 +276,22 @@ def assemble(path: Path, template: bool = False) -> list[Step]: def build(path: Path, out: Path, template: bool = False) -> str: steps = assemble(path, template) - if os.path.lexists(out): - raise Refused("the output directory already exists") - partial = out.with_name(f".{out.name}.partial-{os.getpid()}") - partial.mkdir() + try: + out.mkdir() # an exclusive claim: refuses an existing path, never replaces one + except FileExistsError: + raise Refused("the output directory already exists") from None lines = [] for step in steps: for suffix, body in ((".sh", step.text), (".stdout", "".join(line + "\n" for line in step.stdout))): raw = body.encode("utf-8") - (partial / (step.name + suffix)).write_bytes(raw) + with open(out / (step.name + suffix), "xb") as stream: + stream.write(raw) lines.append(f"{hashlib.sha256(raw).hexdigest()} {step.name}{suffix}\n") manifest = "".join(lines).encode("utf-8") - (partial / "MANIFEST.sha256").write_bytes(manifest) - os.rename(partial, out) # refuses if OUT appeared meanwhile; the partial stays visible + # Written last: a package without its manifest is visibly incomplete. + with open(out / "MANIFEST.sha256", "xb") as stream: + stream.write(manifest) return hashlib.sha256(manifest).hexdigest() diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py index fe7887f1..451fc506 100644 --- a/tests/test_m8_procedure_package.py +++ b/tests/test_m8_procedure_package.py @@ -121,9 +121,15 @@ def test_build_writes_exact_step_bytes_and_a_manifest_atomically(tmp_path): for line in manifest.decode().splitlines(): expected, name = line.split(" ") assert hashlib.sha256((out / name).read_bytes()).hexdigest() == expected - assert "\r" not in script and not list(tmp_path.glob(".package.partial-*")) + assert "\r" not in script with pytest.raises(package.Refused, match="already exists"): package.build(procedure(tmp_path, QUALIFICATION), out) + # An empty directory that appeared first is never replaced. + claimed = tmp_path / "claimed" + claimed.mkdir() + with pytest.raises(package.Refused, match="already exists"): + package.build(procedure(tmp_path, QUALIFICATION), claimed) + assert list(claimed.iterdir()) == [] @pytest.mark.parametrize("change", [ @@ -187,6 +193,8 @@ def test_a_changed_or_crlf_frozen_source_is_refused(tmp_path): "exactly a literal or a fence selection"), (('{literal = "W=/home/m8-service/m8-maint-q12"}', '{literal = "true\\r\\n"}'), "not be LF-only"), + (('{literal = "W=/home/m8-service/m8-maint-q12"}', '{literal = " \\n"}'), + "a literal part is empty"), ]) def test_unresolvable_or_ambiguous_compositions_refuse(tmp_path, change, message): with pytest.raises(package.Refused, match=message): @@ -216,6 +224,24 @@ def test_repeated_steps_and_assigned_placeholders_refuse_sentinels_do_not(tmp_pa sentinel = QUALIFICATION.replace( "m8-maint-q12", "m8-maint-q12\\ntest \\\"$B\\\" != /h/UNPREPARED.bundle") assert "UNPREPARED.bundle" in package.assemble(procedure(tmp_path, sentinel))[0].text + for quoted in ('RUN_ID=\\\"UNREVIEWED\\\"', "RUN_ID='UNPREPARED'"): + assigned = QUALIFICATION.replace("m8-maint-q12", f"m8-maint-q12\\n{quoted}") + with pytest.raises(package.Refused, match="unreviewed placeholder"): + package.assemble(procedure(tmp_path, assigned)) + + +@pytest.mark.parametrize("sources", [ + "sources = []", + '[sources.oc]\npath = 7\nsha256 = "x"', + '[sources.self]\npath = "frozen.md"\nsha256 = "x"', +]) +def test_malformed_sources_refuse_without_a_traceback(tmp_path, sources): + document = (f"# P\n\n## Composition\n\n```toml\n{sources}\n" + '[[steps]]\nname = "s"\nstdout = []\nparts = [{literal = "true"}]\n```\n') + path = tmp_path / "procedure.md" + path.write_bytes(document.encode()) + with pytest.raises(package.Refused, match=r"sources must be a table|string path and sha256"): + package.assemble(path) def test_crlf_document_and_missing_composition_refuse(tmp_path): From d39f8f144a7a10cd8f28ee331d6451d41e1b88d4 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:50:59 -0700 Subject: [PATCH 4/7] fix(m8): allowlist only sibling sources; type-check containers; join labels Connector review of 148aace found these, all fixed: - The allowlist matched by basename, so a pinned copy at fake/<name> passed. Only a bare sibling path of the procedure document can now be an allowlisted reviewed source. - steps = 1 and a non-string source selector raised tracebacks. They are now refused. - A label split by a line continuation (q\<newline>11) escaped the label check. Labels are now matched after joining continued lines. Refs #186 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 15 +++++++++++---- tests/test_m8_procedure_package.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py index 575721a0..3c209994 100644 --- a/scripts/m8_procedure_package.py +++ b/scripts/m8_procedure_package.py @@ -190,7 +190,7 @@ def _part(name: str, part: object, sources: dict, paths: dict) -> tuple[str, str if set(part) != {"source", "heading", "fences"}: raise Refused(f"step {name}: a part is exactly a literal or a fence selection") source, heading, selected = part["source"], part["heading"], part["fences"] - if source not in sources or not isinstance(heading, str): + if not isinstance(source, str) or source not in sources or not isinstance(heading, str): raise Refused(f"step {name}: unknown source {source!r}") if not isinstance(selected, list) or not selected or not all( type(index) is int for index in selected): @@ -227,14 +227,20 @@ def assemble(path: Path, template: bool = False) -> list[Step]: raise Refused(f"source {key} does not match its pinned SHA-256") if b"\r" in raw: raise Refused(f"source {key} is not LF-only") - sources[key], paths[key] = raw.decode("utf-8"), Path(source["path"]).name + # Only a sibling of the procedure (a bare file name) can be an allowlisted + # reviewed source; a pinned copy elsewhere never matches the allowlist. + bare = Path(source["path"]).name == source["path"] + sources[key], paths[key] = raw.decode("utf-8"), source["path"] if bare else None labels = spec.get("labels", {}) if not isinstance(labels, dict) or not set(labels) <= {"current", "historical"}: raise Refused("labels has only current and historical lists") current = set(_strings(labels.get("current", []), "labels.current")) historical_labels = set(_strings(labels.get("historical", []), "labels.historical")) steps, names = [], set() - for entry in spec.get("steps", []): + declared_steps = spec.get("steps", []) + if not isinstance(declared_steps, list): + raise Refused("steps must be a list of tables") + for entry in declared_steps: if not isinstance(entry, dict) or not set(entry) <= STEP_KEYS or not { "name", "stdout", "parts"} <= set(entry): raise Refused("a step has exactly name, stdout, parts and optional historical") @@ -257,7 +263,8 @@ def assemble(path: Path, template: bool = False) -> list[Step]: pieces.append(text) text = "".join(pieces) allowed = current | (historical_labels if historical else set()) - stray = sorted({label for label in LABEL.findall(text) if label not in allowed}) + joined = text.replace("\\\n", "") # Bash joins continued lines into one token + stray = sorted({label for label in LABEL.findall(joined) if label not in allowed}) if stray: raise Refused(f"step {name}: undeclared or historical labels {stray}") if not template and PLACEHOLDER.search(text): diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py index 451fc506..fa2e785c 100644 --- a/tests/test_m8_procedure_package.py +++ b/tests/test_m8_procedure_package.py @@ -230,6 +230,34 @@ def test_repeated_steps_and_assigned_placeholders_refuse_sentinels_do_not(tmp_pa package.assemble(procedure(tmp_path, assigned)) +def test_an_allowlisted_name_outside_the_procedure_directory_is_not_reviewed(tmp_path): + """A pinned copy at fake/frozen.md shares the reviewed name, never its standing.""" + (tmp_path / "fake").mkdir() + (tmp_path / "fake" / "frozen.md").write_bytes(FROZEN.encode()) + path = procedure(tmp_path, QUALIFICATION) + path.write_bytes(path.read_bytes().replace(b'path = "frozen.md"', b'path = "fake/frozen.md"')) + with pytest.raises(package.Refused, match="only a reviewed lane-boundary block"): + package.assemble(path) + + +def test_wrong_container_and_selector_types_refuse_without_a_traceback(tmp_path): + scalar = tmp_path / "scalar.md" + scalar.write_bytes(b"# P\n\n## Composition\n\n```toml\nsteps = 1\n```\n") + with pytest.raises(package.Refused, match="steps must be a list"): + package.assemble(scalar) + listed = QUALIFICATION.replace('{source = "oc", heading = "Setup"', + '{source = ["oc"], heading = "Setup"') + with pytest.raises(package.Refused, match="unknown source"): + package.assemble(procedure(tmp_path, listed)) + + +def test_a_label_split_by_a_line_continuation_is_still_seen(tmp_path): + split = QUALIFICATION.replace('"W=/home/m8-service/m8-maint-q12"', + '"W=/home/m8-service/m8-maint-q\\\\\\n11"') + with pytest.raises(package.Refused, match=r"\['q11'\]"): + package.assemble(procedure(tmp_path, split)) + + @pytest.mark.parametrize("sources", [ "sources = []", '[sources.oc]\npath = 7\nsha256 = "x"', From 13d0f09d65a833c5a6b50a39640610385bf86729 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:18:50 -0700 Subject: [PATCH 5/7] fix(m8): join continuations for placeholders; refuse empty fences, double labels Connector review of d39f8f1 found these, all fixed: - Placeholders were checked on unjoined text, so RUN_ID=UNRE\<newline>VIEWED slipped through. - An empty selected bash fence produced a zero-byte step. - A label declared both current and historical was silently current. Refs #186 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 7 ++++++- tests/test_m8_procedure_package.py | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py index 3c209994..1d70c64f 100644 --- a/scripts/m8_procedure_package.py +++ b/scripts/m8_procedure_package.py @@ -203,6 +203,8 @@ def _part(name: str, part: object, sources: dict, paths: dict) -> tuple[str, str lang, body = found[index] if lang != "bash": raise Refused(f"step {name}: {heading!r} fence {index} is not bash") + if not body.strip(): + raise Refused(f"step {name}: {heading!r} fence {index} is empty") text.append(body) return "".join(text), (paths[source], heading) @@ -236,6 +238,9 @@ def assemble(path: Path, template: bool = False) -> list[Step]: raise Refused("labels has only current and historical lists") current = set(_strings(labels.get("current", []), "labels.current")) historical_labels = set(_strings(labels.get("historical", []), "labels.historical")) + if current & historical_labels: + raise Refused(f"labels {sorted(current & historical_labels)} are both current " + "and historical") steps, names = [], set() declared_steps = spec.get("steps", []) if not isinstance(declared_steps, list): @@ -267,7 +272,7 @@ def assemble(path: Path, template: bool = False) -> list[Step]: stray = sorted({label for label in LABEL.findall(joined) if label not in allowed}) if stray: raise Refused(f"step {name}: undeclared or historical labels {stray}") - if not template and PLACEHOLDER.search(text): + if not template and PLACEHOLDER.search(joined): raise Refused(f"step {name}: an unreviewed placeholder remains") stdout = tuple(_strings(entry["stdout"], f"step {name}: stdout")) rendered = "".join(line + "\n" for line in stdout) diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py index fa2e785c..ddb35159 100644 --- a/tests/test_m8_procedure_package.py +++ b/tests/test_m8_procedure_package.py @@ -251,6 +251,21 @@ def test_wrong_container_and_selector_types_refuse_without_a_traceback(tmp_path) package.assemble(procedure(tmp_path, listed)) +def test_split_placeholders_empty_fences_and_double_labels_refuse(tmp_path): + split = QUALIFICATION.replace("m8-maint-q12", "m8-maint-q12\\nRUN_ID=UNRE\\\\\\nVIEWED") + with pytest.raises(package.Refused, match="unreviewed placeholder"): + package.assemble(procedure(tmp_path, split)) + empty = FROZEN + "\n### Empty\n\n```bash\n\n```\n" + selected = QUALIFICATION.replace('heading = "Setup"', 'heading = "Empty"') + with pytest.raises(package.Refused, match="'Empty' fence 0 is empty"): + package.assemble(procedure(tmp_path, selected, frozen=empty)) + path = procedure(tmp_path, QUALIFICATION) + path.write_bytes(path.read_bytes().replace(b'historical = ["g10", "q11"]', + b'historical = ["g10", "q12"]')) + with pytest.raises(package.Refused, match=r"\['q12'\] are both current and historical"): + package.assemble(path) + + def test_a_label_split_by_a_line_continuation_is_still_seen(tmp_path): split = QUALIFICATION.replace('"W=/home/m8-service/m8-maint-q12"', '"W=/home/m8-service/m8-maint-q\\\\\\n11"') From 8df8e21093781d84bd56f5f0fc7341c3b2bcb1c7 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:45:42 -0700 Subject: [PATCH 6/7] fix(m8): refuse placeholders in any shell spelling; keep literal bytes exact Connector review of 13d0f09 found these, both fixed: - Shell escaping and concatenation ($'X', UNRE\VIEWED, "UNRE""VIEWED") still hid a placeholder. Quotes, backslashes and $ are now removed before matching, and UNREVIEWED anywhere, or UNPREPARED anywhere but the UNPREPARED.bundle sentinel, is refused. - Literal parts lost extra trailing newlines. Their bytes are now kept exactly, and only a missing final newline is supplied. Refs #186 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 17 ++++++++++++----- tests/test_m8_procedure_package.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py index 1d70c64f..ad8574b5 100644 --- a/scripts/m8_procedure_package.py +++ b/scripts/m8_procedure_package.py @@ -40,7 +40,8 @@ comes from an allowlisted, reviewed lane-boundary block; literals never may; - **labels:** an undeclared generation/session label, or a historical one in a step not marked historical. This is a spelling check, not proof of fresh paths; -- **placeholders:** an ``=UNREVIEWED``/``=UNPREPARED`` assignment, unless ``--template``; +- **placeholders:** ``UNREVIEWED`` or ``UNPREPARED`` (other than the + ``UNPREPARED.bundle`` sentinel) in any shell spelling, unless ``--template``; - **bytes:** any CR in a built file, a declared stdout over 65,536 bytes. """ @@ -56,7 +57,11 @@ HEADING = re.compile(r"^(?P<level>#{1,6}) (?P<title>.+?)[ \t]*$") OPEN_FENCE = re.compile(r"^(?P<indent> {0,3})(?P<ticks>`{3,})(?P<lang>[A-Za-z]*)[ \t]*$") LABEL = re.compile(r"(?<![0-9A-Za-z])[gq][0-9]{1,3}(?![0-9A-Za-z])") -PLACEHOLDER = re.compile(r"""=["']?(?:UNREVIEWED|UNPREPARED)\b""") +# Matched after removing quotes, backslashes and `$`, so shell concatenation and +# escaping ($'X', "UNRE""VIEWED", UNRE\VIEWED) cannot hide a placeholder. The +# `UNPREPARED.bundle` refusal sentinel is a comparison target, not a value. +PLACEHOLDER = re.compile(r"\bUNREVIEWED\b|\bUNPREPARED\b(?!\.bundle)") +SHELL_QUOTING = re.compile(r"[\"'\\$]") LANE_NAME = re.compile(r"\bLANE\b") NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") MAX_STDOUT = 65_536 @@ -184,9 +189,11 @@ def _part(name: str, part: object, sources: dict, paths: dict) -> tuple[str, str if not isinstance(part, dict): raise Refused(f"step {name}: a part must be a table") if set(part) == {"literal"} and isinstance(part["literal"], str): - if not part["literal"].strip(): + literal = part["literal"] + if not literal.strip(): raise Refused(f"step {name}: a literal part is empty") - return part["literal"].rstrip("\n") + "\n", None + # Exact bytes: only a missing final newline is supplied. + return literal if literal.endswith("\n") else literal + "\n", None if set(part) != {"source", "heading", "fences"}: raise Refused(f"step {name}: a part is exactly a literal or a fence selection") source, heading, selected = part["source"], part["heading"], part["fences"] @@ -272,7 +279,7 @@ def assemble(path: Path, template: bool = False) -> list[Step]: stray = sorted({label for label in LABEL.findall(joined) if label not in allowed}) if stray: raise Refused(f"step {name}: undeclared or historical labels {stray}") - if not template and PLACEHOLDER.search(joined): + if not template and PLACEHOLDER.search(SHELL_QUOTING.sub("", joined)): raise Refused(f"step {name}: an unreviewed placeholder remains") stdout = tuple(_strings(entry["stdout"], f"step {name}: stdout")) rendered = "".join(line + "\n" for line in stdout) diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py index ddb35159..9f089cc9 100644 --- a/tests/test_m8_procedure_package.py +++ b/tests/test_m8_procedure_package.py @@ -251,6 +251,23 @@ def test_wrong_container_and_selector_types_refuse_without_a_traceback(tmp_path) package.assemble(procedure(tmp_path, listed)) +@pytest.mark.parametrize("spelling", [ + "RUN_ID=$'UNREVIEWED'", "RUN_ID=UNRE\\\\VIEWED", 'RUN_ID=\\"UNRE\\"\\"VIEWED\\"', + "--run UNREVIEWED", "C=UNPREPARED", +]) +def test_every_shell_spelling_of_a_placeholder_refuses(tmp_path, spelling): + escaped = QUALIFICATION.replace("m8-maint-q12", f"m8-maint-q12\\n{spelling}") + with pytest.raises(package.Refused, match="unreviewed placeholder"): + package.assemble(procedure(tmp_path, escaped)) + + +def test_literals_keep_their_exact_trailing_newlines(tmp_path): + exact = QUALIFICATION.replace('{literal = "W=/home/m8-service/m8-maint-q12"}', + '{literal = "W=/home/m8-service/m8-maint-q12\\n\\n\\n"}') + text = package.assemble(procedure(tmp_path, exact))[0].text + assert text.startswith("W=/home/m8-service/m8-maint-q12\n\n\nset -Eeuo pipefail\n") + + def test_split_placeholders_empty_fences_and_double_labels_refuse(tmp_path): split = QUALIFICATION.replace("m8-maint-q12", "m8-maint-q12\\nRUN_ID=UNRE\\\\\\nVIEWED") with pytest.raises(package.Refused, match="unreviewed placeholder"): From 2961af00f3683419b0c4660cb143b627201719b5 Mon Sep 17 00:00:00 2001 From: sushiHex <263549916+sushiHex@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:12:32 -0700 Subject: [PATCH 7/7] fix(m8): exact sentinel token; normalize quoting for labels too Connector review of 8df8e21: - UNPREPARED.bundle.old passed as the sentinel. Only the complete UNPREPARED.bundle token is exempt now. Fixed. - q'1'1 hid a stale label. Labels and placeholders now share one shell-normalized text, with continuations joined and quotes, backslashes and $ removed. Fixed. - ANSI-C hex escapes are recorded as a design choice, not handled. The lints catch honest composition mistakes in reviewed documents; deliberate obfuscation is the review's job, not a shell interpreter's. Refs #186 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S7VMTaJ9XePzc8fjmBBFfs --- scripts/m8_procedure_package.py | 17 +++++++++++------ tests/test_m8_procedure_package.py | 10 +++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/m8_procedure_package.py b/scripts/m8_procedure_package.py index ad8574b5..0ff72ccf 100644 --- a/scripts/m8_procedure_package.py +++ b/scripts/m8_procedure_package.py @@ -57,10 +57,14 @@ HEADING = re.compile(r"^(?P<level>#{1,6}) (?P<title>.+?)[ \t]*$") OPEN_FENCE = re.compile(r"^(?P<indent> {0,3})(?P<ticks>`{3,})(?P<lang>[A-Za-z]*)[ \t]*$") LABEL = re.compile(r"(?<![0-9A-Za-z])[gq][0-9]{1,3}(?![0-9A-Za-z])") -# Matched after removing quotes, backslashes and `$`, so shell concatenation and -# escaping ($'X', "UNRE""VIEWED", UNRE\VIEWED) cannot hide a placeholder. The -# `UNPREPARED.bundle` refusal sentinel is a comparison target, not a value. -PLACEHOLDER = re.compile(r"\bUNREVIEWED\b|\bUNPREPARED\b(?!\.bundle)") +# Matched after removing quotes, backslashes and `$`, so ordinary shell +# concatenation and escaping ($'X', "UNRE""VIEWED", UNRE\VIEWED) cannot hide a +# placeholder or a label. The exact `UNPREPARED.bundle` refusal sentinel is a +# comparison target, not a value. These lints catch honest composition mistakes +# in reviewed documents; deliberate obfuscation (ANSI-C hex escapes, eval, +# indirection) is the review's job, not a shell interpreter's here. +PLACEHOLDER = re.compile(r"\bUNREVIEWED\b|\bUNPREPARED\b(?!\.bundle(?:$|[\s;)|&]))", + re.MULTILINE) SHELL_QUOTING = re.compile(r"[\"'\\$]") LANE_NAME = re.compile(r"\bLANE\b") NAME = re.compile(r"[a-z0-9][a-z0-9-]{0,62}") @@ -275,11 +279,12 @@ def assemble(path: Path, template: bool = False) -> list[Step]: pieces.append(text) text = "".join(pieces) allowed = current | (historical_labels if historical else set()) - joined = text.replace("\\\n", "") # Bash joins continued lines into one token + # Bash joins continued lines and adjacent quoted fragments into one word. + joined = SHELL_QUOTING.sub("", text.replace("\\\n", "")) stray = sorted({label for label in LABEL.findall(joined) if label not in allowed}) if stray: raise Refused(f"step {name}: undeclared or historical labels {stray}") - if not template and PLACEHOLDER.search(SHELL_QUOTING.sub("", joined)): + if not template and PLACEHOLDER.search(joined): raise Refused(f"step {name}: an unreviewed placeholder remains") stdout = tuple(_strings(entry["stdout"], f"step {name}: stdout")) rendered = "".join(line + "\n" for line in stdout) diff --git a/tests/test_m8_procedure_package.py b/tests/test_m8_procedure_package.py index 9f089cc9..1f6d0600 100644 --- a/tests/test_m8_procedure_package.py +++ b/tests/test_m8_procedure_package.py @@ -253,7 +253,8 @@ def test_wrong_container_and_selector_types_refuse_without_a_traceback(tmp_path) @pytest.mark.parametrize("spelling", [ "RUN_ID=$'UNREVIEWED'", "RUN_ID=UNRE\\\\VIEWED", 'RUN_ID=\\"UNRE\\"\\"VIEWED\\"', - "--run UNREVIEWED", "C=UNPREPARED", + "--run UNREVIEWED", "C=UNPREPARED", "RUN_ID=UNPREPARED.bundle.old", + "RUN_ID=UNPREPARED.bundlex", ]) def test_every_shell_spelling_of_a_placeholder_refuses(tmp_path, spelling): escaped = QUALIFICATION.replace("m8-maint-q12", f"m8-maint-q12\\n{spelling}") @@ -283,6 +284,13 @@ def test_split_placeholders_empty_fences_and_double_labels_refuse(tmp_path): package.assemble(path) +def test_a_label_split_by_shell_quoting_is_still_seen(tmp_path): + quoted = QUALIFICATION.replace('"W=/home/m8-service/m8-maint-q12"', + "\"W=/home/m8-service/m8-maint-q'1'1\"") + with pytest.raises(package.Refused, match=r"\['q11'\]"): + package.assemble(procedure(tmp_path, quoted)) + + def test_a_label_split_by_a_line_continuation_is_still_seen(tmp_path): split = QUALIFICATION.replace('"W=/home/m8-service/m8-maint-q12"', '"W=/home/m8-service/m8-maint-q\\\\\\n11"')