From af724ee2f53f4c93fb9c97ac758e5aa7809e4089 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 01:46:09 +0800 Subject: [PATCH 01/17] feat(capture): restore immutable attempt v1 client protocol Source: SpecForge 0b3e00aa291c235e06255fe21b4627d05179e4a8:specforge/inference/capture_protocol.py, copied verbatim. Preserves server-issued attempt fences and publication transfer certificates for native capture integration. --- specforge/inference/capture_protocol.py | 232 ++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 specforge/inference/capture_protocol.py diff --git a/specforge/inference/capture_protocol.py b/specforge/inference/capture_protocol.py new file mode 100644 index 0000000000..c00cde146b --- /dev/null +++ b/specforge/inference/capture_protocol.py @@ -0,0 +1,232 @@ +"""Version-one capture control client and serializable sample writer fences.""" +from __future__ import annotations + +import hashlib +import json +import threading +import time +import uuid +from dataclasses import dataclass, field + + +class CaptureControlRejected(RuntimeError): + """The server explicitly rejected the operation without granting a permit.""" + + +def request_digest(tokens, spec): + value = {"tokens": list(tokens), "features": spec.get("features", {}), + "passthrough": spec.get("passthrough", [])} + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def http_control(url, json_body, timeout): + import requests + response = requests.post(url, json=json_body, timeout=timeout) + response.raise_for_status() + return response.json() + + +class CaptureProtocol: + def __init__(self, base_url, post, timeout): + self.url = base_url.rstrip('/') + '/spec_capture/control' + self.post = post + self.timeout = timeout + self.negotiated = False + self._capability = None + self._capability_known = False + self._capability_lock = threading.RLock() + + def call(self, body): + try: + result = self.post(self.url, json_body={"protocol_version": 1, **body}, timeout=self.timeout) + except Exception as exc: + if body.get('action') != 'capability' and getattr(getattr(exc, 'response', None), 'status_code', None) == 409: + # Refresh discovery only. Never replay an uncertain mutation. + try: + self.capability(refresh=True) + except Exception: + pass # Keep the original rejection recognizable to its owner. + raise + if not isinstance(result, dict) or result.get('error'): + raise RuntimeError('capture control failed: ' + repr(result)) + return result + + def capability(self, *, refresh=False): + with self._capability_lock: + if self._capability_known and not refresh: + return dict(self._capability) if self._capability is not None else None + return self._discover_capability() + + def invalidate_epoch(self, server_epoch): + if self._capability is not None and self._capability['server_epoch'] != server_epoch: + self.capability(refresh=True) + + def _discover_capability(self): + try: + result = self.call({'action': 'capability'}) + except Exception as exc: + status = getattr(getattr(exc, 'response', None), 'status_code', None) + if status == 404 and not self.negotiated: + self._capability = None + self._capability_known = True + return None + raise + if result.get('protocol_version') != 1: + if self.negotiated: + raise RuntimeError('negotiated capture protocol disappeared') + self._capability = None + self._capability_known = True + return None + required = ('server_epoch', 'capture_namespace', 'capture_signature', 'schema', 'cache_enabled') + if any(name not in result for name in required): + raise RuntimeError('incomplete capture capability') + self.negotiated = True + self._capability_known = True + self._capability = dict(result) + return dict(result) + + def begin(self, capability, spec, tokens, attempt_id): + reply = self.call({'action': 'begin', **self._begin_body(capability, spec, tokens, attempt_id)}) + return self._begin_wire(capability, spec, attempt_id, reply) + + def _begin_body(self, capability, spec, tokens, attempt_id): + return {**{name: spec[name] for name in ('store_id', 'sample_id', 'gen')}, + 'server_epoch': capability['server_epoch'], 'attempt_id': attempt_id, + 'request_digest': request_digest(tokens, spec), + 'publication_ownership': capability.get('published_ownership')} + + def begin_batch(self, capability, requests): + if not capability.get('begin_batch'): + return [self.begin(capability, spec, tokens, attempt_id) for spec, tokens, attempt_id in requests] + tasks = [self._begin_body(capability, *request) for request in requests] + result = self.call({'action': 'begin_batch', 'server_epoch': capability['server_epoch'], 'tasks': tasks}) + replies = result.get('attempts') + if not isinstance(replies, list) or len(replies) != len(requests): + raise RuntimeError('capture begin_batch count mismatch') + return [self._begin_wire(capability, spec, attempt_id, reply) + for (spec, _, attempt_id), reply in zip(requests, replies)] + + def _begin_wire(self, capability, spec, attempt_id, reply): + identity = {name: spec[name] for name in ('store_id', 'sample_id', 'gen')} + for name, value in {**identity, 'attempt_id': attempt_id, 'server_epoch': capability['server_epoch']}.items(): + if reply.get(name) != value: + raise RuntimeError('capture begin identity mismatch: ' + name) + if not reply.get('reservation_token'): + raise RuntimeError('capture begin omitted server reservation') + return {**identity, 'protocol_version': 1, 'server_epoch': capability['server_epoch'], + 'attempt_id': attempt_id, 'reservation_token': reply['reservation_token'], + 'control_url': self.url, 'timeout_s': self.timeout, + **({'object_key_layout': 'attempt-v1'} if reply.get('object_key_layout') == 'attempt-v1' else {})} + + +@dataclass +class WriterFence: + """A sample's writer identity; serializable fields travel on every SampleRef.""" + wire: dict + names: tuple + post: object = http_control + writer_drained: bool = False + transferred: bool = False + removed: bool = False + acknowledged: bool = False + adopted: bool = False + cleanup_owner: str = field(default_factory=lambda: uuid.uuid4().hex) + cleanup_token: str | None = None + remote_retired: bool = False + lock: object = field(default_factory=threading.RLock) + + @property + def token(self): + return self.wire['reservation_token'] + + def control(self, action, **values): + body = {name: self.wire[name] for name in ('server_epoch', 'store_id', 'sample_id', 'gen', 'attempt_id', 'reservation_token')} + try: + result = self.post(self.wire['control_url'], json_body={'protocol_version': 1, 'action': action, **body, **values}, + timeout=min(float(self.wire.get('timeout_s', 30.0)), 30.0)) + except Exception as exc: + if getattr(exc, 'capture_control_rejected', False) or getattr(getattr(exc, 'response', None), 'status_code', None) == 409: + raise CaptureControlRejected(str(exc)) from None + raise + if isinstance(result, dict) and result.get('error'): + raise CaptureControlRejected('capture fence control rejected: ' + str(result['error'])) + if not isinstance(result, dict): + raise RuntimeError('capture fence control returned invalid response') + if any(result.get(name) != value for name, value in body.items()): + raise RuntimeError('capture fence identity changed') + return result + + def accept_transfer(self, certificate): + if certificate.get('publication_ownership') != 'attempt-v1': + return False + if self.wire.get('object_key_layout') != 'attempt-v1' or certificate.get('state') != 'PUBLISHED' or certificate.get('writer_drained') is not True: + raise RuntimeError('capture ownership transfer before writer drain') + if any(certificate.get(name) != self.wire[name] for name in + ('server_epoch', 'store_id', 'sample_id', 'gen', 'attempt_id', 'reservation_token')): + raise RuntimeError('capture ownership certificate identity mismatch') + expected = attempt_keys(self.wire, self.names) + if certificate.get('physical_keys') != expected: + raise RuntimeError('capture ownership certificate keys mismatch') + self.wire.update(publication_ownership='attempt-v1', physical_keys=expected, + writer_drained=True, state='PUBLISHED') + self.transferred = self.writer_drained = True + return True + + def drain(self, *, cancel=True): + if self.writer_drained: + return + deadline = time.monotonic() + float(self.wire.get('timeout_s', 30.0)) + while True: + result = self.control('resolve', cancel=cancel) + if result.get('writer_drained'): + self.accept_transfer(result.get("publication_certificate", result)) + self.writer_drained = True + return + if time.monotonic() >= deadline: + raise TimeoutError('capture writer has not drained; sample retained') + time.sleep(0.01) + + def acquire_read(self, reader_owner): + reply = self.control('read_acquire', reader_owner=reader_owner) + if not reply.get('reader_token') or reply.get('reader_owner') != reader_owner: + raise RuntimeError('server did not grant exact read ownership') + return reply['reader_token'] + + def release_read(self, reader_owner, reader_token): + reply = self.control('read_release', reader_owner=reader_owner, reader_token=reader_token) + if not reply.get('reader_released') or reply.get('reader_token') != reader_token: + raise RuntimeError('server did not release exact read ownership') + + def acquire_cleanup(self): + # Only a complete immutable-key ownership certificate bypasses permits. + self.drain(cancel=True) + if self.transferred: + return + reply = self.control('cleanup_acquire', cleanup_owner=self.cleanup_owner) + if reply.get('state') == 'RETIRED': + self.remote_retired = self.removed = self.acknowledged = True + return + if not reply.get('cleanup_token') or reply.get('cleanup_owner') != self.cleanup_owner: + raise RuntimeError('server did not grant exact cleanup ownership') + self.cleanup_token = reply['cleanup_token'] + + def acknowledge(self): + if self.acknowledged: + return + if not self.writer_drained or not self.removed: + raise RuntimeError('cleanup acknowledgement before writer/remove drain') + if self.transferred: + self.acknowledged = True + return + if not self.cleanup_token: + raise RuntimeError('cleanup acknowledgement without server permit') + result = self.control('cleanup_done', cleanup_token=self.cleanup_token) + if result.get('state') != 'RETIRED': + raise RuntimeError('capture cleanup was not retired') + self.acknowledged = True + + +def attempt_keys(wire, names): + """Immutable physical identity; a stale process can only address its attempt.""" + base = f"{wire['store_id']}/{wire['sample_id']}/g{int(wire['gen'])}/attempt/{wire['server_epoch']}/{wire['reservation_token']}" + return {str(name): base + '/' + str(name) for name in names} From a4804d632afdd5253939f59cc91d09a31e115f1c Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 01:46:43 +0800 Subject: [PATCH 02/17] feat(capture): migrate bounded HS cache and assembly controller Extract pure cache/protocol/controller and focused runtime/CUDA ownership tests from frozen 0b3e00aa hs-prefix-cache.patch. Imports now belong to the native capture plugin; no legacy engine capture bridge or backend is restored. Retains atomic coverage leases, actual-storage cache adoption, disjoint pinned arena slots, byte admission, complete feature materialization and quarantine. --- .../specforge_sglang_capture/hs/__init__.py | 45 + .../specforge_sglang_capture/hs/cache.py | 477 +++++++++++ .../specforge_sglang_capture/hs/controller.py | 605 ++++++++++++++ .../specforge_sglang_capture/hs/protocol.py | 251 ++++++ .../test_hs_controller_parallel_cuda.py | 138 ++++ tests/test_runtime/test_hs_runtime.py | 779 ++++++++++++++++++ 6 files changed, 2295 insertions(+) create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py create mode 100644 tests/test_runtime/test_hs_controller_parallel_cuda.py create mode 100644 tests/test_runtime/test_hs_runtime.py diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py new file mode 100644 index 0000000000..2d64e5643e --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py @@ -0,0 +1,45 @@ +"""Single-server content-keyed raw feature cache, frozen Plan A v1 API. + +block_keys(namespace, tokens) returns complete canonical chunk identities. +stats() returns bounded storage/reservation and lifecycle counters. +""" + +from .cache import PrefixFeatureCache +from .controller import CaptureController +from .protocol import ( + AdmissionBudgetError, + CaptureCancelledError, + CaptureCompletion, + CaptureContractError, + CapturePlan, + CapturedSpan, + ContentBlockKey, + DrainHandle, + ExecutionIdentity, + FeatureSchema, + FeatureSpec, + FillReservation, + MaterializedSample, + OutputReservation, + PrefixLease, +) + +__all__ = [ + "AdmissionBudgetError", + "CaptureCancelledError", + "CaptureCompletion", + "CaptureContractError", + "CaptureController", + "CapturePlan", + "CapturedSpan", + "ContentBlockKey", + "DrainHandle", + "ExecutionIdentity", + "FeatureSchema", + "FeatureSpec", + "FillReservation", + "MaterializedSample", + "OutputReservation", + "PrefixFeatureCache", + "PrefixLease", +] diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py new file mode 100644 index 0000000000..99da91219f --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py @@ -0,0 +1,477 @@ +"""Immutable content chunks with atomic continuous coverage and read leases.""" + +from __future__ import annotations + +import hashlib +import struct +import threading +import uuid +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Mapping + +import torch + +from .protocol import ( + CaptureContractError, ContentBlockKey, ExecutionIdentity, FeatureSchema, + FillReservation, PrefixLease, checked_int, _normalized_error, +) + + +def _token_tuple(tokens) -> tuple[int, ...]: + if not isinstance(tokens, tuple): + raise CaptureContractError("tokens must be an immutable tuple") + if any(type(t) is not int or not 0 <= t < 2**63 for t in tokens): + raise CaptureContractError("tokens must be nonnegative canonical int64 ids") + return tokens + + +@dataclass +class _StorageOwner: + nbytes: int + features: dict[str, torch.Tensor] + keys: set[ContentBlockKey] = field(default_factory=set) + leases: int = 0 + + +@dataclass +class _Entry: + key: ContentBlockKey + nbytes: int + state: str = "WRITING" + features: dict[str, torch.Tensor] = field(default_factory=dict) + leases: int = 0 + owner: _StorageOwner | None = None + + +@dataclass +class _Lease: + handle: PrefixLease + namespace: str + entries: tuple[_Entry, ...] + readers: int = 0 + released: bool = False + + +@dataclass +class _Fill: + handle: FillReservation + entry: _Entry + writer_active: bool = False + abort_requested: bool = False + + +class PrefixFeatureCache: + def __init__(self, schema: FeatureSchema, chunk_tokens: int, capacity_bytes: int): + if not isinstance(schema, FeatureSchema): + raise CaptureContractError("cache requires a FeatureSchema") + self.schema = schema + self.chunk_tokens = checked_int(chunk_tokens, "chunk_tokens", 1) + self.capacity_bytes = checked_int(capacity_bytes, "capacity_bytes") + self._chunk_bytes = self.chunk_tokens * self.schema.row_bytes + self._lock = threading.RLock() + self._entries: dict[ContentBlockKey, _Entry] = {} + self._lru: OrderedDict[ContentBlockKey, None] = OrderedDict() + self._leases: dict[str, _Lease] = {} + self._fills: dict[str, _Fill] = {} + self._bytes = self._peak_bytes = self._evictions = self._fill_skips = 0 + self._logical_peak_bytes = self._adopted_samples = self._adoption_skips = 0 + self._owned_bytes = self._owned_peak_bytes = self._owned_groups = 0 + + def _root_digest(self, namespace: str) -> str: + if not isinstance(namespace, str) or not namespace: + raise CaptureContractError("namespace must be a nonempty server-validated string") + encoded = namespace.encode("utf-8") + return hashlib.sha256(b"spec-prefix-root-v1\0" + struct.pack(" str: + encoded = namespace.encode("utf-8") + try: + parent_bytes = bytes.fromhex(parent) + except (TypeError, ValueError) as exc: + raise CaptureContractError("invalid parent SHA256") from exc + if len(parent_bytes) != 32 or len(parent) != 64: + raise CaptureContractError("parent must be a complete SHA256") + body = (b"spec-prefix-block-v1\0" + struct.pack(" tuple[ContentBlockKey, ...]: + """Complete chunks only; no persistent lookup-side token table.""" + _token_tuple(tokens) + parent = self._root_digest(namespace) + result = [] + for start in range(0, len(tokens) - self.chunk_tokens + 1, self.chunk_tokens): + end = start + self.chunk_tokens + chunk = tokens[start:end] + digest = self._digest(namespace, parent, start, end, chunk) + result.append(ContentBlockKey(namespace, start, end, digest, parent, chunk)) + parent = digest + return tuple(result) + + def _validate_key(self, key: ContentBlockKey) -> None: + if not isinstance(key, ContentBlockKey): + raise CaptureContractError("fill needs a content block key") + self._root_digest(key.namespace) + checked_int(key.start, "block start") + checked_int(key.end, "block end") + if key.start % self.chunk_tokens or key.end - key.start != self.chunk_tokens: + raise CaptureContractError("only aligned complete chunks can be cached") + _token_tuple(key._tokens) + if len(key._tokens) != self.chunk_tokens: + raise CaptureContractError("block tokens do not match its complete range") + if key.start == 0 and key.parent_digest != self._root_digest(key.namespace): + raise CaptureContractError("first block has an invalid root identity") + if key.digest != self._digest(key.namespace, key.parent_digest, key.start, key.end, key._tokens): + raise CaptureContractError("content key does not match its tokens/range") + + def _complete(self, entry: _Entry) -> bool: + if entry.state != "READY": + return False + try: + self.schema.validate(entry.features, self.chunk_tokens, "cpu") + return True + except CaptureContractError: + return False + + def lookup_and_pin(self, namespace: str, tokens: tuple[int, ...], limit: int) -> PrefixLease: + _token_tuple(tokens) + checked_int(limit, "coverage limit") + keys = self.block_keys(namespace, tokens[: min(len(tokens), limit)]) + with self._lock: + entries = [] + for key in keys: + entry = self._entries.get(key) + if entry is None or not self._complete(entry): + break + entries.append(entry) + # The query and every refcount update are one metadata transaction. + for entry in entries: + entry.leases += 1 + if entry.owner is not None: + entry.owner.leases += 1 + self._lru.move_to_end(entry.key) + end = len(entries) * self.chunk_tokens + handle = PrefixLease(uuid.uuid4().hex, end) + self._leases[handle.lease_id] = _Lease(handle, namespace, tuple(entries)) + return handle + + def _checked_lease(self, lease: PrefixLease, end: int, + namespace: str | None = None, tokens: tuple[int, ...] | None = None) -> _Lease: + checked_int(end, "lease read end") + rec = self._leases.get(getattr(lease, "lease_id", None)) + if rec is None or rec.handle is not lease or rec.released: + raise CaptureContractError("lease is stale, foreign, or released") + if end > lease.coverage_end: + raise CaptureContractError("lease does not cover the requested prefix") + if namespace is not None and namespace != rec.namespace: + raise CaptureContractError("lease namespace differs from plan") + if tokens is not None: + expected = self.block_keys(rec.namespace, tokens) + needed = (end + self.chunk_tokens - 1) // self.chunk_tokens + if tuple(e.key for e in rec.entries[:needed]) != expected[:needed]: + raise CaptureContractError("lease content differs from plan") + return rec + + def _validate_plan_lease(self, lease, end, namespace, tokens) -> None: + with self._lock: + self._checked_lease(lease, end, namespace, tokens) + + def shrink_lease(self, lease: PrefixLease, end: int) -> PrefixLease: + with self._lock: + rec = self._checked_lease(lease, end) + if rec.readers: + raise CaptureContractError("cannot shrink a lease while a reader is active") + keep = (end + self.chunk_tokens - 1) // self.chunk_tokens + for entry in rec.entries[keep:]: + entry.leases -= 1 + if entry.owner is not None: + entry.owner.leases -= 1 + self._leases.pop(lease.lease_id) + handle = PrefixLease(uuid.uuid4().hex, end) + self._leases[handle.lease_id] = _Lease(handle, rec.namespace, rec.entries[:keep]) + return handle + + def _finish_release(self, rec: _Lease) -> None: + for entry in rec.entries: + entry.leases -= 1 + if entry.owner is not None: + entry.owner.leases -= 1 + assert entry.owner.leases >= 0 + assert entry.leases >= 0 + self._leases.pop(rec.handle.lease_id, None) + + def release_lease(self, lease: PrefixLease) -> None: + with self._lock: + rec = self._leases.get(getattr(lease, "lease_id", None)) + if rec is None or rec.handle is not lease or rec.released: + return + rec.released = True + if not rec.readers: + self._finish_release(rec) + + def copy_prefix_into(self, lease: PrefixLease, end: int, + dest: Mapping[str, torch.Tensor]) -> None: + with self._lock: + rec = self._checked_lease(lease, end) + if set(dest) != {f.name for f in self.schema.features}: + raise CaptureContractError("destination does not contain the complete schema") + rows = {int(t.shape[0]) if isinstance(t, torch.Tensor) and t.ndim == 2 else -1 for t in dest.values()} + if len(rows) != 1 or next(iter(rows)) < end: + raise CaptureContractError("destination cannot hold the leased prefix") + self.schema.validate(dest, next(iter(rows)), "cpu") + entries = rec.entries[: (end + self.chunk_tokens - 1) // self.chunk_tokens] + if any(not self._complete(e) for e in entries): + raise CaptureContractError("leased cache block is missing or corrupt") + rec.readers += 1 + failure = None + try: + spans = [] + for entry in entries: + start, stop = entry.key.start, min(end, entry.key.end) + if entry.owner is not None and spans and spans[-1][2] is entry.owner: + old_start, _, owner, _ = spans[-1] + spans[-1] = (old_start, stop, owner, None) + else: + spans.append((start, stop, entry.owner, entry.features)) + with torch.no_grad(): + for start, stop, owner, block in spans: + for spec in self.schema.features: + source = (owner.features[spec.name][start:stop] if owner is not None + else block[spec.name][:stop - start]) + dest[spec.name][start:stop].copy_(source) + # Drop source views before releasing the reader's lease protection. + source = owner = block = None + spans.clear() + except BaseException as exc: + failure = _normalized_error(exc) + finally: + source = owner = block = None + spans.clear() + with self._lock: + rec.readers -= 1 + if rec.released and not rec.readers: + self._finish_release(rec) + if failure is not None: + raise failure from None + + def reserve_fill(self, key: ContentBlockKey, + execution: ExecutionIdentity) -> FillReservation | None: + self._validate_key(key) + if not isinstance(execution, ExecutionIdentity): + raise CaptureContractError("fill requires an execution identity") + with self._lock: + if key in self._entries or self._chunk_bytes > self.capacity_bytes: + self._fill_skips += 1 + return None + if not self._make_room_locked(self._chunk_bytes): + self._fill_skips += 1 + return None + handle = FillReservation(uuid.uuid4().hex, key, execution, self._chunk_bytes) + entry = _Entry(key, self._chunk_bytes) + self._entries[key] = entry + self._fills[handle.reservation_id] = _Fill(handle, entry) + self._bytes += self._chunk_bytes + self._peak_bytes = max(self._peak_bytes, self._bytes) + self._record_logical_peak_locked() + return handle + + def _record_logical_peak_locked(self): + self._logical_peak_bytes = max(self._logical_peak_bytes, + len(self._entries) * self._chunk_bytes) + + def _remove_entry_locked(self, entry): + entry.features.clear() + self._entries.pop(entry.key, None) + self._lru.pop(entry.key, None) + owner = entry.owner + if owner is None: + self._bytes -= entry.nbytes + else: + owner.keys.discard(entry.key) + if not owner.keys: + # Drop the final owner references before making storage budget + # available. A chunk view never charges only its logical rows. + owner.features.clear() + self._bytes -= owner.nbytes + self._owned_bytes -= owner.nbytes + self._owned_groups -= 1 + entry.owner = None + + def _make_room_locked(self, amount): + if amount > self.capacity_bytes: + return False + while self._bytes + amount > self.capacity_bytes: + victim = None + for key in self._lru: + entry = self._entries[key] + if entry.leases: + continue + if entry.owner is not None and entry.owner.leases: + continue + victim = entry + break + if victim is None: + return False + # Evict an entire owner group. Partial eviction cannot free its + # backing allocation and only creates holes in prefix coverage. + keys = tuple(victim.owner.keys) if victim.owner is not None else (victim.key,) + for key in keys: + self._remove_entry_locked(self._entries[key]) + self._evictions += 1 + return True + + def _adopt_released_sample(self, keys, features) -> bool: + """Take exclusive CPU ownership after every MaterializedSample reader ends. + + Only the controller calls this internal path. Key hashes are prepared + before release. No CPU tensor allocation/copy occurs here; every chunk + view belongs to a group charged for its complete underlying storages, + including any reused prefix, padding, and incomplete tail. + """ + if not keys: + return False + rows = next(iter(features.values())).shape[0] + self.schema.validate(features, rows, "cpu") + storages = {(str(t.device), t.untyped_storage().data_ptr()): t.untyped_storage().nbytes() + for t in features.values()} + nbytes = sum(storages.values()) + owner = _StorageOwner(nbytes, dict(features)) + pending = [] + failure = None + published = charged = False + entry = tensor = None + try: + with self._lock: + if not any(key not in self._entries for key in keys): + self._adoption_skips += 1 + self._fill_skips += len(keys) + elif not self._make_room_locked(nbytes): + self._adoption_skips += 1 + self._fill_skips += len(keys) + else: + # Capacity eviction may itself have removed these keys. + # Register every view with pending before constructing the + # next one, so partial metadata errors drop all references. + for key in keys: + if key not in self._entries: + entry = _Entry(key, self._chunk_bytes, "READY", owner=owner) + pending.append(entry) + for name, tensor in features.items(): + entry.features[name] = tensor[key.start:key.end] + owner.keys.update(entry.key for entry in pending) + self._bytes += nbytes + self._peak_bytes = max(self._peak_bytes, self._bytes) + self._owned_bytes += nbytes + self._owned_groups += 1 + charged = True + self._owned_peak_bytes = max(self._owned_peak_bytes, self._owned_bytes) + for entry in pending: + self._entries[entry.key] = entry + self._lru[entry.key] = None + self._record_logical_peak_locked() + self._adopted_samples += 1 + self._fill_skips += len(keys) - len(pending) + published = True + except BaseException as exc: + failure = _normalized_error(exc) + finally: + tensor = entry = None + if not published: + with self._lock: + for entry in pending: + if self._entries.get(entry.key) is entry: + self._entries.pop(entry.key) + self._lru.pop(entry.key, None) + entry.features.clear() + entry.owner = None + if charged: + self._bytes -= nbytes + self._owned_bytes -= nbytes + self._owned_groups -= 1 + pending.clear() + owner.features.clear() + owner.keys.clear() + entry = None + if failure is not None: + raise failure from None + return published + + def _drop_fill(self, rec: _Fill) -> None: + if self._entries.get(rec.entry.key) is rec.entry: + self._remove_entry_locked(rec.entry) + self._fills.pop(rec.handle.reservation_id, None) + + def publish_ready(self, fill: FillReservation, + features: Mapping[str, torch.Tensor]) -> bool: + with self._lock: + rec = self._fills.get(getattr(fill, "reservation_id", None)) + if rec is None or rec.handle is not fill or rec.writer_active or rec.abort_requested: + return False + rec.writer_active = True + owned = {} + tensor = None + error = None + try: + self.schema.validate(features, self.chunk_tokens, "cpu") + with torch.no_grad(): + for spec in self.schema.features: + tensor = torch.empty((self.chunk_tokens, spec.width), dtype=features[spec.name].dtype, device="cpu") + tensor.copy_(features[spec.name].detach()) + owned[spec.name] = tensor + # No caller tensor/view becomes durable cache storage. + except BaseException as exc: + error = exc + # A caught exception traceback can retain a failed copy's arguments. + # Normalize it and drop all owned locals before reusing WRITING bytes. + failure = _normalized_error(error) if error is not None else None + error = None + tensor = None + with self._lock: + rec.writer_active = False + if failure is not None or rec.abort_requested: + owned.clear() + self._drop_fill(rec) + if failure is not None: + raise failure from None + return False + rec.entry.features = owned + rec.entry.state = "READY" + self._fills.pop(fill.reservation_id) + self._lru[fill.key] = None + return True + + def abort_fill(self, fill: FillReservation) -> None: + with self._lock: + rec = self._fills.get(getattr(fill, "reservation_id", None)) + if rec is None or rec.handle is not fill: + return + if rec.writer_active: + rec.abort_requested = True + else: + self._drop_fill(rec) + + def stats(self) -> dict[str, int]: + with self._lock: + ready_bytes = sum(e.nbytes for e in self._entries.values() + if e.state == "READY" and e.owner is None) + ready_bytes += self._owned_bytes + return { + "ready_blocks": sum(e.state == "READY" for e in self._entries.values()), + "writing_blocks": sum(e.state == "WRITING" for e in self._entries.values()), + "ready_bytes": ready_bytes, + "writing_bytes": sum(e.nbytes for e in self._entries.values() if e.state == "WRITING"), + "cache_bytes": self._bytes, "cache_peak_bytes": self._peak_bytes, + "cache_logical_bytes": len(self._entries) * self._chunk_bytes, + "cache_logical_peak_bytes": self._logical_peak_bytes, + "cache_owned_bytes": self._owned_bytes, + "cache_owned_groups": self._owned_groups, + "cache_owned_peak_bytes": self._owned_peak_bytes, + "cache_adopted_samples": self._adopted_samples, + "cache_adoption_skips": self._adoption_skips, + "active_leases": len(self._leases), "fill_reservations": len(self._fills), + "leased_blocks": sum(e.leases > 0 for e in self._entries.values()), + "evictions": self._evictions, "fill_skips": self._fill_skips, + } diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py new file mode 100644 index 0000000000..46ff01cae1 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py @@ -0,0 +1,605 @@ +"""Bounded capture ownership, asynchronous D2H, full assembly and safe drain. + +B hands over compact independently owned suffix snapshots after admission. +Workers never touch a Req, execute a collective, or release sink-owned output. +""" + +from __future__ import annotations + +import threading +import uuid +from collections import deque +from concurrent.futures import Future, ThreadPoolExecutor +from time import perf_counter +from dataclasses import dataclass, field + +import torch + +from .cache import PrefixFeatureCache, _token_tuple +from .protocol import ( + DTYPES, DTYPE_BYTES, AdmissionBudgetError, CaptureCancelledError, CaptureCompletion, + CaptureContractError, CapturePlan, CapturedSpan, DrainHandle, ExecutionIdentity, + ContentBlockKey, FeatureSchema, MaterializedSample, OutputReservation, PrefixLease, + canonical_digest, checked_int, _normalized_error, +) + + +@dataclass +class _Reservation: + handle: OutputReservation + submitted: bool = False + transient_released: bool = False + release_requested: bool = False + pinned_offset: int = 0 + + +@dataclass +class _Execution: + identity: ExecutionIdentity + reservations: set[str] = field(default_factory=set) + plan: CapturePlan | None = None + cancelled: bool = False + reason: str = "" + jobs: int = 0 + sample: MaterializedSample | None = None + owned_output: dict[str, torch.Tensor] = field(default_factory=dict) + cache_keys: tuple[ContentBlockKey, ...] = () + fills: dict[str, object] = field(default_factory=dict) + quarantined: list[object] = field(default_factory=list) + completion_pending: bool = False + drained_queued: bool = False + drained_polled: bool = False + drained: threading.Event = field(default_factory=threading.Event) + + +@dataclass +class _Job: + plan: CapturePlan + features: dict[str, torch.Tensor] + ready_event: object | None + is_cuda: bool + state: _Execution + future: Future + + +class _UnsafeDrain(RuntimeError): + """The runtime must retain storage when CUDA/producer drain is unproven.""" + + def __init__(self, error, retained): + super().__init__(f"capture storage quarantined: drain could not be proven: {error}") + self.retained = retained + + +class CaptureController: + def __init__(self, cache: PrefixFeatureCache, schema: FeatureSchema, + pinned_budget_bytes: int = 256 * 1024**2, + output_budget_bytes: int = 512 * 1024**2, + device_budget_bytes: int = 256 * 1024**2, *, + max_workers: int = 4): + if not isinstance(cache, PrefixFeatureCache) or schema != cache.schema: + raise CaptureContractError("controller/cache schema mismatch") + self.cache, self.schema = cache, schema + self._limits = { + "pinned": checked_int(pinned_budget_bytes, "pinned_budget_bytes"), + "output": checked_int(output_budget_bytes, "output_budget_bytes"), + "device": checked_int(device_budget_bytes, "device_budget_bytes"), + } + # PyTorch rounds pinned allocations up to a power of two and retains + # freed size buckets. One owned arena prevents cumulative idle buckets + # from bypassing the configured hard bound. Default budgets are exact + # powers of two; other budgets conservatively expose a smaller arena. + self._pinned_configured_bytes = self._limits["pinned"] + self._limits["pinned"] = (1 << (pinned_budget_bytes.bit_length() - 1) + if pinned_budget_bytes else 0) + self._pinned_arena = None + self._slot_alignment = max(DTYPE_BYTES[f.dtype] for f in schema.features) + self._pinned_free = [(0, self._limits["pinned"])] + self._max_workers = checked_int(max_workers, "max_workers", 1) + self._arena_poisoned = False + self._current = dict.fromkeys(self._limits, 0) + self._peaks = dict.fromkeys(self._limits, 0) + self._lock = threading.RLock() + self._reservations: dict[str, _Reservation] = {} + self._executions: dict[ExecutionIdentity, _Execution] = {} + self._completed: deque[CaptureCompletion] = deque() + self._drained: deque[ExecutionIdentity] = deque() + self._max_pending = max(1, min(4096, output_budget_bytes // schema.row_bytes)) + self._closed = False + self._fill_failures = 0 + # Admission attempts include retries; these are not unique-request counts. + # Reasons may overlap when more than one budget is currently exhausted. + self._admission_stats = dict.fromkeys(( + "admission_attempts", "admission_reserved_attempts", + "admission_budget_declined_attempts", "admission_hard_oversize_attempts", + "admission_declined_pinned_attempts", "admission_declined_output_attempts", + "admission_declined_device_attempts", "admission_declined_controller_max_pending_attempts", + "admission_declined_fragmentation_attempts", + ), 0) + self._adoption_calls = 0 + self._adoption_total_ms = self._adoption_max_ms = 0.0 + self._executor = ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="spec-feature-capture") + + def _staging_layout(self, rows): + offset, layout = 0, {} + for spec in self.schema.features: + itemsize = DTYPE_BYTES[spec.dtype] + offset = ((offset + itemsize - 1) // itemsize) * itemsize + size = rows * spec.width * itemsize + layout[spec.name] = (offset, size, spec) + offset += size + offset = ((offset + self._slot_alignment - 1) // self._slot_alignment) * self._slot_alignment + return offset, layout + + def _staging_views(self, rows, reservation=None): + required, layout = self._staging_layout(rows) + if required > self._limits["pinned"]: + raise AdmissionBudgetError("aligned suffix exceeds pinned arena capacity") + with self._lock: + if self._arena_poisoned: + raise CaptureContractError("pinned arena quarantined after unproven DMA drain") + offset = 0 + if reservation is not None: + rec = self._reservation(reservation) + if rec.transient_released or required > reservation.pinned_bytes: + raise CaptureContractError("staging slot is released or too small") + offset = rec.pinned_offset + elif self._current["pinned"]: + raise CaptureContractError("unreserved staging view while slots are active") + # Allocate once under the lock: workers must never race to create + # two retained power-of-two pinned allocations. + if self._pinned_arena is None: + self._pinned_arena = torch.empty((self._limits["pinned"],), dtype=torch.uint8, + device="cpu", pin_memory=True) + return {name: self._pinned_arena[offset + start:offset + start + size] + .view(DTYPES[spec.dtype]).view(rows, spec.width) + for name, (start, size, spec) in layout.items()} + + def _allocate_slot_locked(self, amount): + if not amount: + return 0 + for i, (start, size) in enumerate(self._pinned_free): + if size >= amount: + if size == amount: + self._pinned_free.pop(i) + else: + self._pinned_free[i] = (start + amount, size - amount) + return start + return None + + def _free_slot_locked(self, start, amount): + if not amount: + return + self._pinned_free.append((start, amount)) + merged = [] + for offset, size in sorted(self._pinned_free): + if merged and merged[-1][0] + merged[-1][1] == offset: + old, length = merged[-1] + merged[-1] = (old, length + size) + else: + merged.append((offset, size)) + self._pinned_free = merged + + def try_reserve_output(self, execution: ExecutionIdentity, N: int, L: int, + passthrough_bytes: int) -> OutputReservation | None: + if not isinstance(execution, ExecutionIdentity): + raise CaptureContractError("reservation requires execution identity") + checked_int(N, "N", 1) + checked_int(L, "L") + checked_int(passthrough_bytes, "passthrough_bytes") + if L > N: + raise CaptureContractError("L exceeds N") + charges = {"output": N * self.schema.row_bytes + passthrough_bytes, + "pinned": self._staging_layout(N - L)[0], + "device": (N - L) * self.schema.row_bytes} + with self._lock: + self._admission_stats["admission_attempts"] += 1 + for kind, amount in charges.items(): + if amount > self._limits[kind]: + self._admission_stats["admission_hard_oversize_attempts"] += 1 + raise AdmissionBudgetError(f"request needs {amount} {kind} bytes; hard limit {self._limits[kind]}") + if self._closed: + raise CaptureContractError("capture controller is shut down") + if self._arena_poisoned: + raise CaptureContractError("capture staging quarantined; restart required") + if execution in self._executions: + raise CaptureContractError("execution identity already active; retry needs a new execution epoch") + reasons = [kind for kind in charges + if self._current[kind] + charges[kind] > self._limits[kind]] + if len(self._executions) >= self._max_pending: + reasons.append("controller_max_pending") + if reasons: + self._admission_stats["admission_budget_declined_attempts"] += 1 + for reason in reasons: + self._admission_stats[f"admission_declined_{reason}_attempts"] += 1 + return None + slot = self._allocate_slot_locked(charges["pinned"]) + if slot is None: + self._admission_stats["admission_budget_declined_attempts"] += 1 + self._admission_stats["admission_declined_fragmentation_attempts"] += 1 + return None # Fragmentation cannot authorize overlapping DMA slots. + handle = OutputReservation(uuid.uuid4().hex, execution, charges["output"], + charges["pinned"], charges["device"], N, L, passthrough_bytes) + self._reservations[handle.reservation_id] = _Reservation(handle, pinned_offset=slot) + state = _Execution(execution, {handle.reservation_id}) + self._executions[execution] = state + self._admission_stats["admission_reserved_attempts"] += 1 + for kind, amount in charges.items(): + self._current[kind] += amount + self._peaks[kind] = max(self._peaks[kind], self._current[kind]) + return handle + + def _reservation(self, handle: OutputReservation) -> _Reservation: + rec = self._reservations.get(getattr(handle, "reservation_id", None)) + if rec is None or rec.handle is not handle: + raise CaptureContractError("output reservation is foreign, stale or released") + return rec + + def freeze_plan(self, execution: ExecutionIdentity, namespace: str, tokens: tuple[int, ...], + N: int, L: int, lease: PrefixLease, + reservation: OutputReservation) -> CapturePlan: + _token_tuple(tokens) + if len(tokens) != N: + raise CaptureContractError("plan tokens do not match N") + self.cache._validate_plan_lease(lease, L, namespace, tokens) + digest = canonical_digest(["spec-plan-v1", execution.server_epoch, execution.request_id, + execution.attempt_id, execution.execution_epoch, namespace, self.schema.signature, + N, L, tokens]) + with self._lock: + rec = self._reservation(reservation) + state = self._executions.get(execution) + if (reservation.execution != execution or reservation.N != N or reservation.L != L + or state is None or state.cancelled or rec.submitted): + raise CaptureContractError("plan/reservation execution or range mismatch") + if state.plan is not None: + raise CaptureContractError("execution already has a frozen plan") + plan = CapturePlan(execution, namespace, tokens, N, L, self.schema.signature, + lease, reservation, digest) + state.plan = plan + return plan + + def _release_transient_locked(self, rec: _Reservation) -> None: + if rec.transient_released: + return + rec.transient_released = True + self._free_slot_locked(rec.pinned_offset, rec.handle.pinned_bytes) + self._current["pinned"] -= rec.handle.pinned_bytes + self._current["device"] -= rec.handle.device_bytes + + def _release_host_locked(self, rec: _Reservation, state: _Execution) -> None: + self._current["output"] -= rec.handle.host_bytes + self._reservations.pop(rec.handle.reservation_id, None) + state.reservations.discard(rec.handle.reservation_id) + assert all(v >= 0 for v in self._current.values()) + + def _maybe_drained_locked(self, state: _Execution) -> None: + if state.jobs or state.reservations or state.fills or state.quarantined: + return + state.drained.set() + if not state.drained_queued: + self._drained.append(state.identity) + state.drained_queued = True + self._reap_locked(state) + + def _reap_locked(self, state: _Execution) -> None: + if state.drained_polled and state.drained.is_set() and not state.completion_pending: + if self._executions.get(state.identity) is state: + self._executions.pop(state.identity) + + def release_output(self, reservation: OutputReservation) -> None: + with self._lock: + rec = self._reservations.get(getattr(reservation, "reservation_id", None)) + if rec is None or rec.handle is not reservation: + return + rec.release_requested = True + if rec.submitted: + # The worker/sink remains the owner. No timeout can revoke its read. + return + state = self._executions[reservation.execution] + if state.plan is not None: + self.cache.release_lease(state.plan.lease) + self._release_transient_locked(rec) + self._release_host_locked(rec, state) + self._maybe_drained_locked(state) + + def _sample_released(self, reservation: OutputReservation) -> None: + with self._lock: + rec = self._reservations.get(reservation.reservation_id) + if rec is None or rec.handle is not reservation: + return + state = self._executions[reservation.execution] + state.sample = None + owned, state.owned_output = state.owned_output, {} + adoption_started = perf_counter() + try: + # MaterializedSample.release has already removed the sink's + # tensor references. Readers may not access or modify them + # after this ownership transfer. Keep the output charge until + # the cache has charged every retained backing allocation. + if owned and not state.cancelled and not rec.release_requested and not self._closed: + self._adoption_calls += 1 + self.cache._adopt_released_sample(state.cache_keys, owned) + except Exception: + self._fill_failures += 1 + finally: + elapsed_ms = (perf_counter() - adoption_started) * 1000.0 + self._adoption_total_ms += elapsed_ms + self._adoption_max_ms = max(self._adoption_max_ms, elapsed_ms) + state.cache_keys = () + owned.clear() + self._release_transient_locked(rec) + self._release_host_locked(rec, state) + self._maybe_drained_locked(state) + + def submit(self, plan: CapturePlan, span: CapturedSpan) -> Future[MaterializedSample]: + if not isinstance(plan, CapturePlan) or not isinstance(span, CapturedSpan): + raise CaptureContractError("submit requires a frozen plan and span") + if span.execution != plan.execution or (span.start, span.end) != (plan.L, plan.N): + raise CaptureContractError("captured span must be the plan's absolute [L,N)") + if plan.schema_signature != self.schema.signature: + raise CaptureContractError("plan schema mismatch") + features = dict(span.features) + self.schema.validate(features, plan.N - plan.L) + devices = {tensor.device for tensor in features.values()} + if len(devices) != 1 or next(iter(devices)).type not in ("cpu", "cuda"): + raise CaptureContractError("all suffix features must share one CPU/CUDA device") + is_cuda = next(iter(devices)).type == "cuda" + if is_cuda and span.ready_event is None: + raise CaptureContractError("CUDA snapshots require a producer ready event") + if span.ready_event is not None and not callable(getattr(span.ready_event, "synchronize", None)): + raise CaptureContractError("ready_event must support synchronize") + # Count underlying storage, not tensor-view rows. B must hand over compact + # owned snapshots; a tiny suffix view of a full padded batch is rejected. + storages = {(str(t.device), t.untyped_storage().data_ptr()): t.untyped_storage().nbytes() + for t in features.values()} + source_bytes = sum(storages.values()) + if source_bytes > plan.reservation.device_bytes: + raise CaptureContractError("snapshot underlying storage exceeds the admission reservation") + with self._lock: + rec = self._reservation(plan.reservation) + state = self._executions.get(plan.execution) + if (state is None or state.plan is not plan or state.cancelled or rec.release_requested + or rec.submitted or self._closed): + raise CaptureContractError("execution is stale, cancelled, submitted or closed") + self.cache._validate_plan_lease(plan.lease, plan.L, plan.namespace, plan.tokens) + rec.submitted = True + state.jobs += 1 + future = Future() + # Future.cancel() cannot cancel the cleanup worker and strand storage. + future.set_running_or_notify_cancel() + job = _Job(plan, features, span.ready_event, is_cuda, state, future) + try: + self._executor.submit(self._run_job, job) + except BaseException: + rec.submitted = False + state.jobs -= 1 + raise + return future + + def _cancelled(self, job: _Job) -> bool: + with self._lock: + rec = self._reservations.get(job.plan.reservation.reservation_id) + return job.state.cancelled or rec is None or rec.release_requested + + def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: + """Only this worker owns these sources/staging until the real copy drain.""" + if not job.is_cuda: + if job.ready_event is not None: + try: + job.ready_event.synchronize() + except BaseException as exc: + raise _UnsafeDrain(exc, [job.ready_event]) from exc + return {} if self._cancelled(job) else dict(job.features) + staging = {} + stream = None + source = target = None + device = next(iter(job.features.values())).device + try: + with torch.cuda.device(device): + if self._cancelled(job): + job.ready_event.synchronize() + return {} + with self._lock: + poisoned = self._arena_poisoned + if poisoned: + # A previous failed DMA still owns this shared landing + # buffer. Drain this job's producer, but never touch arena. + job.ready_event.synchronize() + raise CaptureContractError("pinned arena quarantined after unproven DMA drain") + stream = torch.cuda.Stream(device=device) + with torch.cuda.stream(stream): + stream.wait_event(job.ready_event) + staging = self._staging_views(job.plan.N - job.plan.L, job.plan.reservation) + for name, source in job.features.items(): + target = staging[name] + target.copy_(source, non_blocking=True) + source.record_stream(stream) + copied = torch.cuda.Event() + copied.record(stream) + copied.synchronize() + return staging + except BaseException as exc: + drain_errors = [] + # A failed wait_event may leave the copy stream independent of the + # producer. Drain BOTH owners, even if either synchronize raises. + if stream is not None: + try: + stream.synchronize() + except BaseException as drain_error: + drain_errors.append(drain_error) + try: + job.ready_event.synchronize() + except BaseException as drain_error: + drain_errors.append(drain_error) + if drain_errors: + raise _UnsafeDrain(drain_errors[0], [staging, stream, job.ready_event]) from exc + source = target = None + staging.clear() + raise + + def _prepare_cache_adoption(self, job: _Job) -> tuple: + # Hash/construct keys on the assembly worker, never in the sink release + # callback (which may run on the scheduler thread). + try: + return self.cache.block_keys(job.plan.namespace, job.plan.tokens) + except Exception: + with self._lock: + self._fill_failures += 1 + return () + + def _run_job(self, job: _Job) -> None: + output, staging = {}, {} + sample = None + completion = None + unsafe = False + try: + copy_started = perf_counter() + staging = self._copy_suffix(job) + copied_at = perf_counter() + job.features.clear() + job.ready_event = None + if self._cancelled(job): + raise CaptureCancelledError(job.state.reason or "execution released") + with torch.no_grad(): + for spec in self.schema.features: + output[spec.name] = torch.empty((job.plan.N, spec.width), dtype=DTYPES[spec.dtype], device="cpu") + try: + self.cache.copy_prefix_into(job.plan.lease, job.plan.L, output) + finally: + self.cache.release_lease(job.plan.lease) + for name in output: + output[name][job.plan.L:job.plan.N].copy_(staging[name]) + staging.clear() + with self._lock: + self._release_transient_locked(self._reservation(job.plan.reservation)) + if self._cancelled(job): + raise CaptureCancelledError(job.state.reason or "execution released") + self.schema.validate(output, job.plan.N, "cpu") + assembled_at = perf_counter() + cache_keys = self._prepare_cache_adoption(job) + filled_at = perf_counter() + sample = MaterializedSample(job.plan.execution, output, job.plan.N, + lambda: self._sample_released(job.plan.reservation), timings={ + "d2h_wait_ms": (copied_at - copy_started) * 1000.0, + "assembly_ms": (assembled_at - copied_at) * 1000.0, + "cache_fill_ms": (filled_at - assembled_at) * 1000.0, + }) + with self._lock: + job.state.owned_output = dict(output) + job.state.cache_keys = cache_keys + output.clear() + if self._cancelled(job): + sample.release() + sample = None + raise CaptureCancelledError(job.state.reason or "execution released") + job.state.sample = sample + completion = CaptureCompletion(job.plan.execution, sample=sample) + except _UnsafeDrain as exc: + unsafe = True + with self._lock: + # A persistent arena must never be reused while any old DMA + # may still access it. Also stop fresh admission immediately. + self._arena_poisoned = True + # Retain sources AND landing buffers. Keep every charge until a + # real recovery mechanism proves drain; timeout is never success. + job.state.quarantined.extend([job, exc.retained]) + completion = CaptureCompletion(job.plan.execution, error=RuntimeError(str(exc))) + except BaseException as exc: + completion = CaptureCompletion(job.plan.execution, error=_normalized_error(exc)) + finally: + self.cache.release_lease(job.plan.lease) + output.clear() + if not unsafe: + staging.clear() + job.features.clear() + job.ready_event = None + with self._lock: + rec = self._reservations.get(job.plan.reservation.reservation_id) + if rec is not None: + self._release_transient_locked(rec) + if sample is None: + self._release_host_locked(rec, job.state) + with self._lock: + job.state.jobs -= 1 + job.state.completion_pending = True + self._completed.append(completion) + self._maybe_drained_locked(job.state) + # Publish the result only after completion bookkeeping and resources. + if completion.error is not None: + job.future.set_exception(completion.error) + else: + job.future.set_result(completion.sample) + + def poll_completed(self) -> list[CaptureCompletion]: + with self._lock: + result = list(self._completed) + self._completed.clear() + for completion in result: + state = self._executions.get(completion.execution) + if state is not None: + state.completion_pending = False + self._reap_locked(state) + return result + + def invalidate(self, execution: ExecutionIdentity, reason: str) -> DrainHandle: + with self._lock: + state = self._executions.get(execution) + if state is None: + event = threading.Event() + event.set() + return DrainHandle(event) + state.cancelled, state.reason = True, str(reason) + for fill in tuple(state.fills.values()): + self.cache.abort_fill(fill) + for token in tuple(state.reservations): + rec = self._reservations.get(token) + if rec is not None and not rec.submitted: + self.release_output(rec.handle) + self._maybe_drained_locked(state) + return DrainHandle(state.drained) + + def poll_drained(self) -> list[ExecutionIdentity]: + with self._lock: + result = list(self._drained) + self._drained.clear() + for execution in result: + state = self._executions.get(execution) + if state is not None: + state.drained_polled = True + self._reap_locked(state) + return result + + def stats(self) -> dict[str, int | float]: + result = self.cache.stats() + with self._lock: + result.update(self._admission_stats) + result.update({f"{k}_bytes": v for k, v in self._current.items()}) + result.update({f"{k}_peak_bytes": v for k, v in self._peaks.items()}) + result.update(pending_executions=len(self._executions), + pending_copies=sum(s.jobs for s in self._executions.values()), + output_reservations=len(self._reservations), + quarantined_executions=sum(bool(s.quarantined) for s in self._executions.values()), + fill_failures=self._fill_failures, + capture_workers=self._max_workers, + cache_adoption_calls=self._adoption_calls, + cache_adoption_total_ms=self._adoption_total_ms, + cache_adoption_max_ms=self._adoption_max_ms, + pinned_free_bytes=sum(size for _, size in self._pinned_free), + pinned_free_slots=len(self._pinned_free)) + result.update(pinned_configured_bytes=self._pinned_configured_bytes, + pinned_capacity_bytes=self._limits["pinned"], + pinned_arena_bytes=(self._pinned_arena.numel() if self._pinned_arena is not None else 0), + pinned_arena_poisoned=int(self._arena_poisoned)) + return result + + def shutdown(self, wait: bool = True) -> None: + with self._lock: + self._closed = True + executions = tuple(self._executions) + for execution in executions: + self.invalidate(execution, "controller shutdown") + # Never cancel queued cleanup workers. They retain producer events/storage. + self._executor.shutdown(wait=wait, cancel_futures=False) + if wait: + with self._lock: + if any(not state.drained.is_set() for state in self._executions.values()): + raise RuntimeError("capture shutdown not drained: sink output or quarantined storage still owned") diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py new file mode 100644 index 0000000000..2c6feaa553 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py @@ -0,0 +1,251 @@ +"""Plan A v1 values, ownership handles, and canonical feature validation. + +Only aux and post-norm last_hidden artifacts are runtime feature names. Sink +object names belong to the bridge. ContentBlockKey._tokens is frozen per-block +metadata (root-approved v1 clarification), never a wire field or lookup side table. +""" + +from __future__ import annotations + +import hashlib +import json +import threading +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Callable, Mapping + +import torch + +DTYPES = { + "bfloat16": torch.bfloat16, + "float16": torch.float16, + "float32": torch.float32, + "float64": torch.float64, +} +DTYPE_BYTES = {"bfloat16": 2, "float16": 2, "float32": 4, "float64": 8} + + +class CaptureContractError(ValueError): + """A malformed plan/span/schema, rather than a recoverable cache miss.""" + + +class AdmissionBudgetError(CaptureContractError): + """One request cannot fit even when all other reservations are released.""" + + +class CaptureCancelledError(RuntimeError): + """Execution fenced; all safely reclaimable resources have been drained.""" + + +def checked_int(value: int, name: str, minimum: int = 0) -> int: + if type(value) is not int or value < minimum: + raise CaptureContractError(f"{name} must be an integer >= {minimum}") + return value + + +def canonical_digest(value) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("ascii")).hexdigest() + + +def _normalized_error(error: BaseException) -> BaseException: + """A safe error outcome cannot retain drained tensor/stream traceback locals.""" + try: + clean = type(error)(str(error)) + except BaseException: + clean = RuntimeError(f"{type(error).__name__}: {error}") + clean.__traceback__ = None + clean.__cause__ = None + clean.__context__ = None + clean.__suppress_context__ = True + return clean + + +@dataclass(frozen=True) +class FeatureSpec: + name: str + width: int + dtype: str + layer_ids: tuple[int, ...] + + def __post_init__(self): + if self.name not in ("aux", "last_hidden"): + raise CaptureContractError(f"unknown feature artifact {self.name!r}") + checked_int(self.width, "feature width", 1) + if self.dtype not in DTYPES: + raise CaptureContractError(f"unsupported canonical dtype {self.dtype!r}") + if not isinstance(self.layer_ids, tuple): + raise CaptureContractError("layer_ids must be an immutable tuple") + for layer in self.layer_ids: + checked_int(layer, "layer id") + if len(set(self.layer_ids)) != len(self.layer_ids): + raise CaptureContractError("duplicate capture layers") + if self.name == "aux" and not self.layer_ids: + raise CaptureContractError("aux needs ordered capture layer ids") + if self.name == "last_hidden" and self.layer_ids: + raise CaptureContractError("post-norm last_hidden uses layer_ids=()") + + +@dataclass(frozen=True) +class FeatureSchema: + features: tuple[FeatureSpec, ...] + signature: str = field(init=False) + row_bytes: int = field(init=False) + + def __post_init__(self): + if not isinstance(self.features, tuple) or not self.features: + raise CaptureContractError("features must be a nonempty immutable tuple") + if any(not isinstance(f, FeatureSpec) for f in self.features): + raise CaptureContractError("schema requires FeatureSpec entries") + if len({f.name for f in self.features}) != len(self.features): + raise CaptureContractError("duplicate feature artifact") + manifest = [(f.name, f.width, f.dtype, f.layer_ids) for f in self.features] + object.__setattr__(self, "signature", canonical_digest(["spec-features-v1", manifest])) + object.__setattr__(self, "row_bytes", sum(f.width * DTYPE_BYTES[f.dtype] for f in self.features)) + + def validate(self, features: Mapping[str, torch.Tensor], rows: int, + device: str | None = None) -> None: + checked_int(rows, "feature rows") + if set(features) != {f.name for f in self.features}: + raise CaptureContractError("feature names do not match the complete schema") + for spec in self.features: + tensor = features[spec.name] + if not isinstance(tensor, torch.Tensor): + raise CaptureContractError(f"{spec.name} is not a tensor") + if tuple(tensor.shape) != (rows, spec.width): + raise CaptureContractError(f"{spec.name} shape {tuple(tensor.shape)} != {(rows, spec.width)}") + if tensor.dtype != DTYPES[spec.dtype]: + raise CaptureContractError(f"{spec.name} dtype does not match {spec.dtype}") + if device is not None and tensor.device.type != device: + raise CaptureContractError(f"{spec.name} must be on {device}") + if not tensor.is_contiguous(): + raise CaptureContractError(f"{spec.name} must be contiguous") + + +@dataclass(frozen=True) +class ExecutionIdentity: + server_epoch: str + request_id: str + attempt_id: str + execution_epoch: int + + def __post_init__(self): + if not all(isinstance(v, str) and v for v in (self.server_epoch, self.request_id, self.attempt_id)): + raise CaptureContractError("execution identity fields must be nonempty strings") + checked_int(self.execution_epoch, "execution epoch") + + +@dataclass(frozen=True) +class ContentBlockKey: + namespace: str + start: int + end: int + digest: str + parent_digest: str + _tokens: tuple[int, ...] = field(repr=False) + + +@dataclass(frozen=True) +class PrefixLease: + lease_id: str + coverage_end: int + + +@dataclass(frozen=True) +class FillReservation: + reservation_id: str + key: ContentBlockKey + execution: ExecutionIdentity + nbytes: int + + +@dataclass(frozen=True) +class OutputReservation: + reservation_id: str + execution: ExecutionIdentity + host_bytes: int + pinned_bytes: int + device_bytes: int + N: int + L: int + passthrough_bytes: int + + +@dataclass(frozen=True) +class CapturePlan: + execution: ExecutionIdentity + namespace: str + tokens: tuple[int, ...] + N: int + L: int + schema_signature: str + lease: PrefixLease + reservation: OutputReservation + plan_digest: str + + +@dataclass(frozen=True) +class CapturedSpan: + execution: ExecutionIdentity + start: int + end: int + features: Mapping[str, torch.Tensor] + ready_event: torch.cuda.Event | None + + +class MaterializedSample: + """Owned CPU output; release only after every sink reader has ended. + + A release drops this object's tensor references before returning its budget. + Copies/views retained by the caller must have ended before invoking release. + """ + + def __init__(self, execution: ExecutionIdentity, features: Mapping[str, torch.Tensor], + n_tokens: int, on_release: Callable[[], None], *, timings=None): + self.execution = execution + self.n_tokens = n_tokens + self.timings = MappingProxyType(dict(timings or {})) + self._features = MappingProxyType(dict(features)) + self._on_release = on_release + self._lock = threading.Lock() + self._released = False + + @property + def features(self) -> Mapping[str, torch.Tensor]: + with self._lock: + if self._released: + raise CaptureContractError("sample already released") + return self._features + + def release(self) -> None: + with self._lock: + if self._released: + return + self._released = True + self._features = MappingProxyType({}) + callback, self._on_release = self._on_release, lambda: None + callback() + + +@dataclass(frozen=True) +class CaptureCompletion: + execution: ExecutionIdentity + sample: MaterializedSample | None = None + error: BaseException | None = None + + def __post_init__(self): + if (self.sample is None) == (self.error is None): + raise CaptureContractError("completion must have exactly one outcome") + + +class DrainHandle: + """A timeout means storage is still owned, never permission to reclaim.""" + + def __init__(self, event: threading.Event): + self._event = event + + def done(self) -> bool: + return self._event.is_set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) diff --git a/tests/test_runtime/test_hs_controller_parallel_cuda.py b/tests/test_runtime/test_hs_controller_parallel_cuda.py new file mode 100644 index 0000000000..9fc5bac444 --- /dev/null +++ b/tests/test_runtime/test_hs_controller_parallel_cuda.py @@ -0,0 +1,138 @@ +"""Explicitly scheduled real-CUDA checks for concurrent capture landing slots. + +Run on one allocated GPU with SGLANG_RUN_SPEC_FEATURE_CUDA_TESTS=1. The event +wrapper gates only the host's completion observation; all CUDA streams, DMA +copies, and underlying events remain real. CPU discovery never initializes CUDA. +""" + +import json +import os +import threading +import unittest +from unittest import mock + +import torch + +from specforge_sglang_capture.hs import ( + CaptureCancelledError, + CaptureController, + CapturedSpan, + ExecutionIdentity, + FeatureSchema, + FeatureSpec, + PrefixFeatureCache, +) + + +@unittest.skipUnless( + os.environ.get("SGLANG_RUN_SPEC_FEATURE_CUDA_TESTS") == "1", + "requires an explicitly allocated CUDA GPU", +) +class ConcurrentCaptureCudaTests(unittest.TestCase): + def run_pair(self, cancel_first): + self.assertTrue(torch.cuda.is_available()) + schema = FeatureSchema((FeatureSpec("aux", 16, "bfloat16", (1,)), + FeatureSpec("last_hidden", 16, "bfloat16", ()))) + rows = 4096 + sample_bytes = rows * schema.row_bytes + cache = PrefixFeatureCache(schema, 256, sample_bytes * 4) + controller = CaptureController(cache, schema, sample_bytes * 2, + sample_bytes * 4, sample_bytes * 2, max_workers=2) + plans, spans, expected, samples = [], [], [], [] + tokens = tuple(range(rows)) + for i in range(2): + execution = ExecutionIdentity("cuda-probe", str(i), str(i), 0) + namespace = f"cuda-{i}" + lease = cache.lookup_and_pin(namespace, tokens, 0) + reservation = controller.try_reserve_output(execution, rows, 0, 0) + self.assertIsNotNone(reservation) + plans.append(controller.freeze_plan(execution, namespace, tokens, rows, 0, + lease, reservation)) + cpu = ((torch.arange(rows * 16).reshape(rows, 16) % 113) + i).to(torch.bfloat16) + expected.append({"aux": cpu.clone(), "last_hidden": cpu + 5}) + values = {name: tensor.to("cuda") for name, tensor in expected[-1].items()} + producer = torch.cuda.Event() + producer.record(torch.cuda.current_stream()) + spans.append(CapturedSpan(execution, 0, rows, values, producer)) + # Each worker must enqueue its own real D2H before either completion + # observation is allowed to return. This exposes overlapping ownership + # even on devices whose physical DMA engines serialize the transfers. + submitted = threading.Barrier(3) + finish = threading.Event() + real_event = torch.cuda.Event + class CopyEvent: + def __init__(self): + self.event = real_event() + def record(self, stream): + self.event.record(stream) + def synchronize(self): + submitted.wait(10) + if not finish.wait(10): + raise RuntimeError("CUDA test host gate did not open") + self.event.synchronize() + futures = [] + try: + with mock.patch("specforge_sglang_capture.hs.controller.torch.cuda.Event", CopyEvent): + futures = [controller.submit(plan, span) for plan, span in zip(plans, spans)] + spans.clear() + values = None + submitted.wait(10) + self.assertTrue(all(not future.done() for future in futures)) + self.assertEqual(controller.stats()["pinned_bytes"], sample_bytes * 2) + offsets = [controller._reservations[p.reservation.reservation_id].pinned_offset + for p in plans] + self.assertEqual(abs(offsets[1] - offsets[0]), sample_bytes) + extra = ExecutionIdentity("cuda-probe", "extra", "extra", 0) + self.assertIsNone(controller.try_reserve_output(extra, 4, 0, 0)) + drain = None + if cancel_first: + drain = controller.invalidate(plans[0].execution, "real DMA cancellation") + self.assertFalse(drain.done()) + self.assertEqual(controller.stats()["pinned_bytes"], sample_bytes * 2) + finish.set() + for i, future in enumerate(futures): + if cancel_first and i == 0: + with self.assertRaises(CaptureCancelledError): + future.result(10) + else: + sample = future.result(10) + samples.append(sample) + for name, tensor in sample.features.items(): + self.assertTrue(torch.equal(tensor, expected[i][name])) + if drain is not None: + self.assertTrue(drain.wait(2)) + self.assertEqual(controller.stats()["pinned_bytes"], 0) + self.assertEqual(controller.stats()["device_bytes"], 0) + for sample in samples: + sample.release() + self.assertEqual(cache.stats()["ready_blocks"], 16 * (1 if cancel_first else 2)) + stats = controller.stats() + self.assertEqual(stats["output_bytes"], 0) + self.assertEqual(stats["quarantined_executions"], 0) + self.assertEqual(stats["pinned_arena_bytes"], sample_bytes * 2) + self.assertEqual(stats["pinned_free_bytes"], sample_bytes * 2) + print(json.dumps({"real_cuda": True, "cancel_first": cancel_first, + "concurrently_submitted_d2h": 2, "bitexact": True, + "stats": stats})) + finally: + finish.set() + for future in futures: + try: + future.result(15).release() + except Exception: + pass + for sample in samples: + sample.release() + controller.poll_completed() + controller.poll_drained() + controller.shutdown() + + def test_two_real_d2h_jobs_are_isolated_and_bitexact(self): + self.run_pair(False) + + def test_cancelled_real_d2h_slot_stays_owned_until_observed_drain(self): + self.run_pair(True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_runtime.py b/tests/test_runtime/test_hs_runtime.py new file mode 100644 index 0000000000..adb8601a3c --- /dev/null +++ b/tests/test_runtime/test_hs_runtime.py @@ -0,0 +1,779 @@ +"""CPU-only adversarial tests of the production Plan A cache/controller.""" + +import gc +import threading +import weakref +from contextlib import nullcontext +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from unittest import mock + +import torch + +from specforge_sglang_capture.hs import ( + AdmissionBudgetError, CaptureCancelledError, CaptureCompletion, + CaptureContractError, CaptureController, CapturedSpan, ExecutionIdentity, + FeatureSchema, FeatureSpec, PrefixFeatureCache, +) + + +def schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)), + FeatureSpec("last_hidden", 2, "float32", ()))) + + +def identity(n=0, epoch=0): + return ExecutionIdentity("server", f"request-{n}", f"attempt-{n}", epoch) + + +def features(n, start=0): + rows = torch.arange(start * 2, (start + n) * 2, dtype=torch.float32).reshape(n, 2) + return {"aux": rows.clone(), "last_hidden": (rows + 1000).clone()} + + +def populate(cache, tokens, values=None, namespace="ns"): + values = features(len(tokens)) if values is None else values + for key in cache.block_keys(namespace, tokens): + fill = cache.reserve_fill(key, identity()) + if fill is not None: + cache.publish_ready(fill, {k: v[key.start:key.end] for k, v in values.items()}) + + +class GateEvent: + def __init__(self): + self.entered = threading.Event() + self.finished = threading.Event() + + def synchronize(self): + self.entered.set() + if not self.finished.wait(5): + raise RuntimeError("test producer event was not completed") + + +class CacheTests(unittest.TestCase): + def setUp(self): + self.cache = PrefixFeatureCache(schema(), 4, 128) + self.tokens = tuple(range(8)) + + def test_schema_order_and_shape_are_semantic(self): + a = FeatureSchema((FeatureSpec("aux", 2, "float32", (3, 1)),)) + b = FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)),)) + self.assertNotEqual(a.signature, b.signature) + self.assertEqual(schema().row_bytes, 16) + for kwargs in [dict(name="unknown", width=2, dtype="float32", layer_ids=()), + dict(name="aux", width=2, dtype="torch.float32", layer_ids=(1,)), + dict(name="last_hidden", width=2, dtype="float32", layer_ids=(1,))]: + with self.assertRaises(CaptureContractError): + FeatureSpec(**kwargs) + with self.assertRaises(CaptureContractError): + schema().validate({"aux": torch.zeros(4, 2)}, 4) + + def test_keys_bind_entire_prefix_namespace_and_chunk_size(self): + same_tail = (9, 10, 11, 12) + self.tokens[4:] + keys = self.cache.block_keys("ns", self.tokens) + self.assertEqual(len(keys[0].digest), 64) + self.assertNotEqual(keys[1].digest, self.cache.block_keys("ns", same_tail)[1].digest) + self.assertNotEqual(keys[0].digest, self.cache.block_keys("other", self.tokens)[0].digest) + self.assertNotEqual(keys[0].digest, PrefixFeatureCache(schema(), 2, 128).block_keys("ns", self.tokens)[0].digest) + self.assertEqual(keys, self.cache.block_keys("ns", self.tokens)) + for tokens in [(True, 1, 2, 3), (-1, 1, 2, 3), (2**63, 1, 2, 3)]: + with self.assertRaises(CaptureContractError): + self.cache.block_keys("ns", tokens) + with self.assertRaises(CaptureContractError): + self.cache.reserve_fill(replace(keys[0], _tokens=(9, 9, 9, 9)), identity()) + + def test_hole_and_writing_stop_contiguous_coverage(self): + first, second = self.cache.block_keys("ns", self.tokens) + pending = self.cache.reserve_fill(first, identity()) + later = self.cache.reserve_fill(second, identity(1)) + self.cache.publish_ready(later, features(4, 4)) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + self.assertEqual(lease.coverage_end, 0) + self.cache.release_lease(lease) + self.cache.publish_ready(pending, features(4)) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + self.assertEqual(lease.coverage_end, 8) + self.cache.release_lease(lease) + + def test_no_incomplete_tail_or_limit_chunk_reuse(self): + populate(self.cache, self.tokens) + for query, limit, expected in [(self.tokens, 8, 8), (self.tokens, 7, 4), + (self.tokens[:7], 7, 4), (self.tokens[:3], 3, 0)]: + lease = self.cache.lookup_and_pin("ns", query, limit) + self.assertEqual(lease.coverage_end, expected) + self.cache.release_lease(lease) + + def test_missing_feature_publish_rolls_back_without_ready(self): + fill = self.cache.reserve_fill(self.cache.block_keys("ns", self.tokens)[0], identity()) + with self.assertRaises(CaptureContractError): + self.cache.publish_ready(fill, {"aux": torch.zeros(4, 2)}) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.cache.stats()["fill_reservations"], 0) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + self.assertEqual(lease.coverage_end, 0) + self.cache.release_lease(lease) + + def test_shrink_inside_chunk_consumes_handle_and_stale_release_is_safe(self): + populate(self.cache, self.tokens) + old = self.cache.lookup_and_pin("ns", self.tokens, 8) + new = self.cache.shrink_lease(old, 3) + self.cache.release_lease(old) + self.assertEqual(self.cache.stats()["active_leases"], 1) + self.assertEqual(self.cache.stats()["leased_blocks"], 1) + out = features(8, 100) + self.cache.copy_prefix_into(new, 3, out) + self.assertTrue(torch.equal(out["aux"][:3], features(3)["aux"])) + with self.assertRaises(CaptureContractError): + self.cache.copy_prefix_into(old, 3, out) + self.cache.release_lease(new) + self.cache.release_lease(new) + self.assertEqual(self.cache.stats()["active_leases"], 0) + + def test_pinned_blocks_cannot_be_evicted_and_cache_does_not_alias(self): + values = features(8) + populate(self.cache, self.tokens, values) + values["aux"].fill_(-99) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + key = self.cache.block_keys("ns", (99, 98, 97, 96))[0] + self.assertIsNone(self.cache.reserve_fill(key, identity(1))) + out = features(8) + self.cache.copy_prefix_into(lease, 8, out) + self.assertTrue(torch.equal(out["aux"], features(8)["aux"])) + out["aux"].fill_(-999) + self.cache.copy_prefix_into(lease, 8, out) + self.assertTrue(torch.equal(out["aux"], features(8)["aux"])) + self.cache.release_lease(lease) + fill = self.cache.reserve_fill(key, identity(1)) + self.assertIsNotNone(fill) + self.assertLessEqual(self.cache.stats()["cache_bytes"], 128) + self.cache.abort_fill(fill) + + def test_single_committer_and_foreign_abort_do_not_erase_ready(self): + key = self.cache.block_keys("ns", self.tokens)[0] + with ThreadPoolExecutor(8) as pool: + results = list(pool.map(lambda n: self.cache.reserve_fill(key, identity(n)), range(8))) + winners = [r for r in results if r is not None] + self.assertEqual(len(winners), 1) + owner = winners[0] + self.cache.abort_fill(replace(owner)) + self.assertEqual(self.cache.stats()["writing_blocks"], 1) + self.assertTrue(self.cache.publish_ready(owner, features(4))) + self.cache.abort_fill(owner) + self.assertFalse(self.cache.publish_ready(owner, features(4))) + self.assertEqual(self.cache.stats()["ready_blocks"], 1) + + def test_lookup_pin_is_atomic_against_eviction(self): + cache = PrefixFeatureCache(schema(), 4, 64) + populate(cache, self.tokens[:4]) + entered, resume = threading.Event(), threading.Event() + complete = cache._complete + def blocked(entry): + entered.set() + self.assertTrue(resume.wait(3)) + return complete(entry) + with mock.patch.object(cache, "_complete", blocked), ThreadPoolExecutor(2) as pool: + lookup = pool.submit(cache.lookup_and_pin, "ns", self.tokens[:4], 4) + self.assertTrue(entered.wait(3)) + key = cache.block_keys("ns", (8, 9, 10, 11))[0] + eviction = pool.submit(cache.reserve_fill, key, identity(2)) + resume.set() + lease = lookup.result(3) + self.assertEqual(lease.coverage_end, 4) + self.assertIsNone(eviction.result(3)) + cache.release_lease(lease) + + def test_abort_during_cpu_writer_keeps_reservation_until_drain(self): + fill = self.cache.reserve_fill(self.cache.block_keys("ns", self.tokens)[0], identity()) + entered, resume = threading.Event(), threading.Event() + real_empty = torch.empty + def blocked(*args, **kwargs): + entered.set() + self.assertTrue(resume.wait(3)) + return real_empty(*args, **kwargs) + values = features(4) + with mock.patch("specforge_sglang_capture.hs.cache.torch.empty", blocked), ThreadPoolExecutor(1) as pool: + writer = pool.submit(self.cache.publish_ready, fill, values) + self.assertTrue(entered.wait(3)) + self.cache.abort_fill(fill) + self.assertEqual(self.cache.stats()["cache_bytes"], 64) + self.assertEqual(self.cache.stats()["fill_reservations"], 1) + resume.set() + self.assertFalse(writer.result(3)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + + def test_release_during_cpu_reader_defers_reclaim(self): + cache = PrefixFeatureCache(schema(), 4, 64) + populate(cache, self.tokens[:4]) + lease = cache.lookup_and_pin("ns", self.tokens[:4], 4) + dest = features(4) + entered, resume = threading.Event(), threading.Event() + real_copy = torch.Tensor.copy_ + def blocked(tensor, *args, **kwargs): + entered.set() + self.assertTrue(resume.wait(3)) + return real_copy(tensor, *args, **kwargs) + with mock.patch.object(torch.Tensor, "copy_", blocked), ThreadPoolExecutor(1) as pool: + reader = pool.submit(cache.copy_prefix_into, lease, 4, dest) + self.assertTrue(entered.wait(3)) + cache.release_lease(lease) + key = cache.block_keys("ns", (8, 9, 10, 11))[0] + self.assertIsNone(cache.reserve_fill(key, identity(2))) + resume.set() + reader.result(3) + fill = cache.reserve_fill(key, identity(2)) + self.assertIsNotNone(fill) + cache.abort_fill(fill) + self.assertEqual(cache.stats()["active_leases"], 0) + + + def test_failed_cache_copy_does_not_retain_owned_storage_via_traceback(self): + fill = self.cache.reserve_fill(self.cache.block_keys("ns", self.tokens)[0], identity()) + values = features(4) + real_empty, real_copy = torch.empty, torch.Tensor.copy_ + allocated, retained_errors = [], [] + count = [0] + def allocate(*args, **kwargs): + tensor = real_empty(*args, **kwargs) + allocated.append(weakref.ref(tensor)) + return tensor + def fail_second(tensor, *args, **kwargs): + count[0] += 1 + if count[0] == 2: + raise RuntimeError(tensor) # Deliberately retain the tensor in args/traceback. + return real_copy(tensor, *args, **kwargs) + with mock.patch("specforge_sglang_capture.hs.cache.torch.empty", allocate), \ + mock.patch.object(torch.Tensor, "copy_", fail_second): + try: + self.cache.publish_ready(fill, values) + except RuntimeError as error: + retained_errors.append(error) + gc.collect() + self.assertEqual(len(retained_errors), 1) + self.assertTrue(all(ref() is None for ref in allocated)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + + def test_owner_group_charges_entire_storage_and_any_lease_blocks_eviction(self): + cache = PrefixFeatureCache(schema(), 4, 160) + tokens = tuple(range(9)) + values = features(9) + self.assertTrue(cache._adopt_released_sample(cache.block_keys("ns", tokens), values)) + values.clear() # Ownership has transferred; callers retain no aliases. + self.assertEqual(cache.stats()["cache_bytes"], 144) + self.assertEqual(cache.stats()["cache_logical_bytes"], 128) + lease = cache.lookup_and_pin("ns", tokens, 4) + different = cache.block_keys("other", (9, 8, 7, 6))[0] + self.assertIsNone(cache.reserve_fill(different, identity(2))) + self.assertEqual(cache.stats()["ready_blocks"], 2) + self.assertEqual(cache.stats()["cache_bytes"], 144) + out = features(4, 100) + cache.copy_prefix_into(lease, 4, out) + self.assertTrue(torch.equal(out["aux"], features(4)["aux"])) + cache.release_lease(lease) + fill = cache.reserve_fill(different, identity(2)) + self.assertIsNotNone(fill) + self.assertEqual(cache.stats()["ready_blocks"], 0) + self.assertEqual(cache.stats()["cache_owned_bytes"], 0) + self.assertEqual(cache.stats()["cache_bytes"], 64) + cache.abort_fill(fill) + + def test_contiguous_owner_prefix_uses_one_copy_per_feature(self): + cache = PrefixFeatureCache(schema(), 4, 160) + tokens = tuple(range(9)) + values = features(9) + cache._adopt_released_sample(cache.block_keys("ns", tokens), values) + values.clear() + lease = cache.lookup_and_pin("ns", tokens, 8) + out = features(8, 100) + copies = [] + real_copy = torch.Tensor.copy_ + def copy(target, source, *args, **kwargs): + copies.append(tuple(target.shape)) + return real_copy(target, source, *args, **kwargs) + with mock.patch.object(torch.Tensor, "copy_", copy): + cache.copy_prefix_into(lease, 8, out) + cache.release_lease(lease) + self.assertEqual(copies, [(8, 2), (8, 2)]) + self.assertTrue(torch.equal(out["aux"], features(8)["aux"])) + + def test_adoption_counts_hidden_backing_storage_and_rejects_oversize(self): + values = {name: tensor[:4] for name, tensor in features(9).items()} + keys = self.cache.block_keys("ns", self.tokens[:4]) + self.assertFalse(self.cache._adopt_released_sample(keys, values)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + larger = PrefixFeatureCache(schema(), 4, 160) + self.assertTrue(larger._adopt_released_sample(keys, values)) + self.assertEqual(larger.stats()["cache_bytes"], 144) + self.assertEqual(larger.stats()["cache_logical_bytes"], 64) + + + +class ControllerTests(unittest.TestCase): + def setUp(self): + self.cache = PrefixFeatureCache(schema(), 4, 1024) + self.controller = CaptureController(self.cache, schema(), 512, 512, 512) + self.samples = [] + + def tearDown(self): + for sample in self.samples: + sample.release() + self.controller.poll_completed() + self.controller.poll_drained() + self.controller.shutdown() + + def plan(self, n=8, L=0, execution=None, namespace="ns", passthrough=0): + execution = identity() if execution is None else execution + tokens = tuple(range(n)) + lease = self.cache.lookup_and_pin(namespace, tokens, n) + self.assertGreaterEqual(lease.coverage_end, L) + lease = self.cache.shrink_lease(lease, L) + reservation = self.controller.try_reserve_output(execution, n, L, passthrough) + self.assertIsNotNone(reservation) + return self.controller.freeze_plan(execution, namespace, tokens, n, L, lease, reservation) + + def submit(self, plan, event=None): + span = CapturedSpan(plan.execution, plan.L, plan.N, features(plan.N-plan.L, plan.L), event) + return self.controller.submit(plan, span) + + def take(self, future): + sample = future.result(5) + self.samples.append(sample) + return sample + + def test_full_sample_from_non_aligned_prefix_and_suffix(self): + cold = self.take(self.submit(self.plan(n=9))) + self.assertTrue(torch.equal(cold.features["aux"], features(9)["aux"])) + self.assertEqual(self.cache.stats()["ready_blocks"], 0) + cold.release() + self.assertEqual(self.cache.stats()["ready_blocks"], 2) + self.controller.poll_completed() + self.controller.poll_drained() + plan = self.plan(n=9, L=5, execution=identity(1)) + warm = self.take(self.submit(plan)) + for name in warm.features: + self.assertTrue(torch.equal(warm.features[name], features(9)[name])) + warm.features["aux"].fill_(-99) + lease = self.cache.lookup_and_pin("ns", tuple(range(9)), 9) + out = features(9) + self.cache.copy_prefix_into(lease, 8, out) + self.cache.release_lease(lease) + self.assertTrue(torch.equal(out["aux"][:8], features(8)["aux"])) + self.assertEqual(self.controller.stats()["active_leases"], 0) + + def test_output_charge_survives_completion_and_cancel_until_sink_release(self): + self.controller = CaptureController(self.cache, schema(), 512, 128, 512) + sample = self.take(self.submit(self.plan())) + self.assertEqual(self.controller.stats()["output_bytes"], 128) + self.assertEqual(self.controller.stats()["pinned_bytes"], 0) + self.assertIsNone(self.controller.try_reserve_output(identity(1), 8, 0, 0)) + drain = self.controller.invalidate(identity(), "HTTP timeout after publication started") + self.assertFalse(drain.wait(0.01)) + self.assertEqual(sample.features["aux"].shape, (8, 2)) + sample.release() + self.assertTrue(drain.wait(1)) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + with self.assertRaises(CaptureContractError): + _ = sample.features + + def test_hard_budget_oversize_and_passthrough(self): + for args in [(100, 0, 0), (8, 0, 500)]: + with self.assertRaises(AdmissionBudgetError): + self.controller.try_reserve_output(identity(), *args) + self.assertEqual(self.controller.stats()["output_reservations"], 0) + res = self.controller.try_reserve_output(identity(), 8, 0, 17) + self.assertEqual(res.host_bytes, 145) + self.controller.release_output(res) + self.controller.release_output(res) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_pinned_and_device_pressure_are_independent_hard_limits(self): + for limits in [(128, 1024, 1024), (1024, 1024, 128)]: + with self.subTest(limits=limits): + controller = CaptureController(self.cache, schema(), *limits) + reserved = controller.try_reserve_output(identity(30), 8, 0, 0) + self.assertIsNone(controller.try_reserve_output(identity(31), 4, 0, 0)) + self.assertEqual(controller.stats()["pinned_bytes"], 128) + self.assertEqual(controller.stats()["device_bytes"], 128) + with self.assertRaises(AdmissionBudgetError): + controller.try_reserve_output(identity(32), 9, 0, 0) + controller.release_output(reserved) + self.assertEqual(controller.stats()["output_bytes"], 0) + controller.shutdown() + + def test_admission_stats_count_attempt_reasons_and_separate_hard_oversize(self): + for kind, limits in (('pinned', (128, 1024, 1024)), + ('output', (1024, 128, 1024)), + ('device', (1024, 1024, 128))): + with self.subTest(kind=kind): + controller = CaptureController(self.cache, schema(), *limits) + try: + held = controller.try_reserve_output(identity(70), 8, 0, 0) + for _ in range(2): + self.assertIsNone(controller.try_reserve_output(identity(71), 4, 0, 0)) + with self.assertRaises(AdmissionBudgetError): + controller.try_reserve_output(identity(72), 100, 0, 0) + stats = controller.stats() + self.assertEqual(stats['admission_attempts'], 4) + self.assertEqual(stats['admission_reserved_attempts'], 1) + self.assertEqual(stats['admission_budget_declined_attempts'], 2) + self.assertEqual(stats[f'admission_declined_{kind}_attempts'], 2) + self.assertEqual(stats['admission_hard_oversize_attempts'], 1) + self.assertEqual(stats['admission_declined_controller_max_pending_attempts'], 0) + controller.release_output(held) + finally: + controller.shutdown() + + def test_controller_pending_and_fragmentation_have_distinct_admission_reasons(self): + self.controller._max_pending = 1 + held = self.controller.try_reserve_output(identity(80), 1, 0, 0) + self.assertIsNone(self.controller.try_reserve_output(identity(81), 1, 0, 0)) + self.controller.release_output(held) + self.controller.poll_drained() + self.assertEqual(self.controller.stats()['admission_declined_controller_max_pending_attempts'], 1) + self.controller._max_pending = 4096 + # Four 128-byte slots fill the 512-byte arena. Two separated free + # slots have enough total bytes for 192, but no contiguous DMA slot. + slots = [self.controller.try_reserve_output(identity(90 + i), 8, 0, 0) for i in range(4)] + self.controller.release_output(slots[0]) + self.controller.release_output(slots[2]) + self.controller.poll_drained() + self.assertIsNone(self.controller.try_reserve_output(identity(94), 12, 0, 0)) + stats = self.controller.stats() + self.assertEqual(stats['admission_declined_fragmentation_attempts'], 1) + self.assertEqual(stats['admission_declined_pinned_attempts'], 0) + self.assertEqual(stats['admission_declined_output_attempts'], 0) + self.assertEqual(stats['admission_declined_device_attempts'], 0) + for slot in slots: + self.controller.release_output(slot) + + def test_pinned_arena_is_one_retained_bounded_allocation_across_sizes(self): + real_empty = torch.empty + allocations = [] + def allocate(*args, **kwargs): + self.assertTrue(kwargs.pop("pin_memory")) + value = real_empty(*args, **kwargs) + allocations.append(value) + return value + with mock.patch("torch.empty", allocate): + for rows in (1, 8, 3, 16, 2): + views = self.controller._staging_views(rows) + for spec in schema().features: + value = views[spec.name] + self.assertEqual(tuple(value.shape), (rows, spec.width)) + self.assertEqual(value.untyped_storage().data_ptr(), + self.controller._pinned_arena.untyped_storage().data_ptr()) + self.assertEqual(len(allocations), 1) + self.assertEqual(allocations[0].numel(), 512) + self.assertEqual(self.controller.stats()["pinned_arena_bytes"], 512) + self.assertEqual(self.controller.stats()["pinned_bytes"], 0) + + def test_pinned_arena_capacity_is_conservative_and_alignment_is_charged(self): + controller = CaptureController(self.cache, schema(), 96, 1024, 1024) + self.assertEqual(controller.stats()["pinned_capacity_bytes"], 64) + with self.assertRaises(AdmissionBudgetError): + controller.try_reserve_output(identity(60), 5, 0, 0) + controller.shutdown() + mixed = FeatureSchema((FeatureSpec("aux", 1, "bfloat16", (1,)), + FeatureSpec("last_hidden", 1, "float32", ()))) + controller = CaptureController(PrefixFeatureCache(mixed, 1, 64), mixed, 16, 64, 64) + reservation = controller.try_reserve_output(identity(61), 1, 0, 0) + self.assertEqual(reservation.pinned_bytes, 8) + self.assertEqual(reservation.device_bytes, 6) + controller.release_output(reservation) + controller.shutdown() + + def test_quarantined_arena_forbids_new_reservation_and_landing_views(self): + self.controller._arena_poisoned = True + with self.assertRaisesRegex(CaptureContractError, "quarantined"): + self.controller.try_reserve_output(identity(62), 1, 0, 0) + with self.assertRaisesRegex(CaptureContractError, "quarantined"): + self.controller._staging_views(1) + + def test_admission_rollback_releases_frozen_lease_and_all_budgets(self): + populate(self.cache, tuple(range(8))) + plan = self.plan(L=4) + self.controller.release_output(plan.reservation) + self.cache.release_lease(plan.lease) + for key in ("output_bytes", "pinned_bytes", "device_bytes", "active_leases"): + self.assertEqual(self.controller.stats()[key], 0) + with self.assertRaises(CaptureContractError): + self.submit(plan) + + def test_cancel_waits_producer_event_and_future_cannot_cancel_cleanup(self): + plan = self.plan() + gate = GateEvent() + future = self.submit(plan, gate) + self.assertTrue(gate.entered.wait(3)) + self.assertFalse(future.cancel()) + drain = self.controller.invalidate(plan.execution, "retracted") + self.assertFalse(drain.done()) + self.assertEqual(self.controller.stats()["device_bytes"], 128) + gate.finished.set() + with self.assertRaises(CaptureCancelledError): + future.result(3) + self.assertTrue(drain.wait(1)) + for key in ("output_bytes", "pinned_bytes", "device_bytes", "active_leases", "pending_copies"): + self.assertEqual(self.controller.stats()[key], 0) + self.assertEqual(self.cache.stats()["ready_blocks"], 0) + + def test_old_epoch_completion_cannot_complete_new_execution(self): + old = self.plan(execution=identity(0, 0)) + gate = GateEvent() + old_future = self.submit(old, gate) + self.assertTrue(gate.entered.wait(3)) + self.controller.invalidate(old.execution, "retract") + new = self.plan(execution=identity(0, 1)) + new_future = self.submit(new) + gate.finished.set() + with self.assertRaises(CaptureCancelledError): + old_future.result(3) + self.take(new_future) + completions = self.controller.poll_completed() + self.assertEqual({c.execution for c in completions}, {old.execution, new.execution}) + self.assertIsInstance(next(c.error for c in completions if c.execution == old.execution), CaptureCancelledError) + self.assertIsNotNone(next(c.sample for c in completions if c.execution == new.execution)) + + def test_wrong_span_or_underlying_storage_is_rejected_before_submit(self): + plan = self.plan() + with self.assertRaises(CaptureContractError): + self.controller.submit(plan, CapturedSpan(plan.execution, 1, 8, features(7), None)) + too_large = features(8) + too_large["aux"] = torch.arange(1000, dtype=torch.float32)[:16].reshape(8, 2) + with self.assertRaises(CaptureContractError): + self.controller.submit(plan, CapturedSpan(plan.execution, 0, 8, too_large, None)) + self.assertEqual(self.controller.stats()["pending_copies"], 0) + self.controller.release_output(plan.reservation) + + def test_worker_failure_drains_and_future_reports_original_error(self): + plan = self.plan() + with mock.patch.object(self.controller, "_copy_suffix", side_effect=RuntimeError("D2H submission failed")): + future = self.submit(plan) + with self.assertRaisesRegex(RuntimeError, "D2H submission failed"): + future.result(3) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + self.assertEqual(self.controller.stats()["device_bytes"], 0) + self.assertEqual(self.cache.stats()["active_leases"], 0) + completions = self.controller.poll_completed() + self.assertEqual(len(completions), 1) + self.assertIsNotNone(completions[0].error) + + def test_cache_adoption_failure_keeps_complete_sample_delivery(self): + sample = self.take(self.submit(self.plan())) + self.assertTrue(torch.equal(sample.features["aux"], features(8)["aux"])) + with mock.patch.object(self.cache, "_adopt_released_sample", side_effect=RuntimeError("cache metadata failed")): + sample.release() + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.controller.stats()["fill_failures"], 1) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_cache_capacity_declines_adoption_without_blocking_release(self): + self.cache.capacity_bytes = 64 + sample = self.take(self.submit(self.plan())) + sample.release() + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.cache.stats()["cache_adoption_skips"], 1) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_release_transfers_same_storage_without_allocating_or_copying(self): + sample = self.take(self.submit(self.plan(n=9))) + pointers = {name: tensor.untyped_storage().data_ptr() for name, tensor in sample.features.items()} + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.controller.stats()["output_bytes"], 144) + original = self.cache._adopt_released_sample + def adopt(keys, values): + # The output charge remains live throughout cache admission. + self.assertEqual(self.controller.stats()["output_bytes"], 144) + return original(keys, values) + with mock.patch.object(self.cache, "_adopt_released_sample", adopt), \ + mock.patch("torch.empty", side_effect=AssertionError("release allocated a tensor")), \ + mock.patch.object(torch.Tensor, "copy_", side_effect=AssertionError("release copied a tensor")): + sample.release() + self.assertEqual(self.controller.stats()["output_bytes"], 0) + self.assertEqual(self.cache.stats()["cache_bytes"], 144) + self.assertEqual(self.cache.stats()["cache_logical_bytes"], 128) + self.assertEqual(self.cache.stats()["cache_owned_groups"], 1) + for entry in self.cache._entries.values(): + for name, tensor in entry.features.items(): + self.assertEqual(tensor.untyped_storage().data_ptr(), pointers[name]) + with self.assertRaises(CaptureContractError): + _ = sample.features + + def test_cancelled_sample_does_not_transfer_output_to_cache(self): + plan = self.plan() + sample = self.take(self.submit(plan)) + drain = self.controller.invalidate(plan.execution, "retract after assembly") + self.assertFalse(drain.done()) + sample.release() + self.assertTrue(drain.wait(1)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + + def test_no_complete_chunk_does_not_retain_full_output(self): + sample = self.take(self.submit(self.plan(n=3))) + references = [weakref.ref(tensor) for tensor in sample.features.values()] + sample.release() + gc.collect() + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertTrue(all(reference() is None for reference in references)) + + def test_workers_assemble_independent_jobs_concurrently(self): + gate = GateEvent() + first = self.submit(self.plan(execution=identity(60)), gate) + self.assertTrue(gate.entered.wait(3)) + second = self.submit(self.plan(execution=identity(61))) + second_sample = self.take(second) + self.assertFalse(first.done()) + self.assertTrue(torch.equal(second_sample.features["aux"], features(8)["aux"])) + gate.finished.set() + self.take(first) + + def test_reserved_slots_are_disjoint_and_coalesce_only_after_release(self): + first = self.controller.try_reserve_output(identity(70), 8, 0, 0) + second = self.controller.try_reserve_output(identity(71), 8, 0, 0) + real_empty = torch.empty + def allocate(*args, **kwargs): + kwargs.pop("pin_memory", None) + return real_empty(*args, **kwargs) + with mock.patch("torch.empty", allocate): + a = self.controller._staging_views(8, first) + b = self.controller._staging_views(8, second) + for value in a.values(): + value.fill_(10) + for value in b.values(): + value.fill_(20) + self.assertTrue(all(bool(torch.all(value == 10)) for value in a.values())) + self.assertTrue(all(bool(torch.all(value == 20)) for value in b.values())) + # No reader remains when a slot is explicitly released here. + a.clear() + self.controller.release_output(first) + with self.assertRaises(CaptureContractError): + self.controller._staging_views(8, first) + third = self.controller.try_reserve_output(identity(72), 8, 0, 0) + c = self.controller._staging_views(8, third) + for value in c.values(): + value.fill_(30) + self.assertTrue(all(bool(torch.all(value == 20)) for value in b.values())) + b.clear() + c.clear() + self.controller.release_output(second) + self.controller.release_output(third) + self.assertEqual(self.controller._pinned_free, [(0, 512)]) + + def test_corrupt_lease_fails_instead_of_zero_filling(self): + populate(self.cache, tuple(range(8))) + plan = self.plan(L=4) + entry = self.cache._entries[self.cache.block_keys("ns", tuple(range(8)))[0]] + entry.features.pop("last_hidden") + future = self.submit(plan) + with self.assertRaisesRegex(CaptureContractError, "missing or corrupt"): + future.result(3) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_shutdown_nonblocking_keeps_inflight_storage_until_actual_drain(self): + plan = self.plan() + gate = GateEvent() + future = self.submit(plan, gate) + self.assertTrue(gate.entered.wait(3)) + self.controller.shutdown(wait=False) + self.assertEqual(self.controller.stats()["device_bytes"], 128) + gate.finished.set() + with self.assertRaises(CaptureCancelledError): + future.result(3) + self.controller.shutdown(wait=True) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_partial_copy_and_failed_wait_drain_both_owners_without_traceback_storage(self): + # CPU tensors plus fully mocked CUDA entry points test scheduling/lifetime + # logic deterministically. This is not CUDA numerical/device evidence. + real_copy, real_empty = torch.Tensor.copy_, torch.empty + for fail_wait in (False, True): + with self.subTest(fail_wait=fail_wait): + plan = self.plan(execution=identity(20 + int(fail_wait))) + allocated, retained_errors = [], [] + copies = [0] + class Producer: + syncs = 0 + def synchronize(producer): + producer.syncs += 1 + producer = Producer() + class Stream: + syncs = 0 + def wait_event(stream, event): + if fail_wait: + raise RuntimeError("wait_event failed before dependency") + def synchronize(stream): + stream.syncs += 1 + stream = Stream() + def allocate(*args, **kwargs): + kwargs.pop("pin_memory", None) + tensor = real_empty(*args, **kwargs) + allocated.append(weakref.ref(tensor)) + return tensor + def copy(tensor, *args, **kwargs): + copies[0] += 1 + if copies[0] == 2: + raise RuntimeError(tensor) + return real_copy(tensor, *args, **kwargs) + original = self.controller._copy_suffix + def pretend_cuda(job): + job.is_cuda = True + return original(job) + with mock.patch.object(self.controller, "_copy_suffix", pretend_cuda), \ + mock.patch("torch.cuda.device", return_value=nullcontext()), \ + mock.patch("torch.cuda.stream", return_value=nullcontext()), \ + mock.patch("torch.cuda.Stream", return_value=stream), \ + mock.patch("torch.empty", allocate), \ + mock.patch.object(torch.Tensor, "record_stream", return_value=None), \ + mock.patch.object(torch.Tensor, "copy_", copy): + future = self.submit(plan, producer) + try: + future.result(3) + except RuntimeError as error: + retained_errors.append(error) + gc.collect() + self.assertEqual(len(retained_errors), 1) + self.assertEqual(producer.syncs, 1) + self.assertEqual(stream.syncs, 1) + # The single explicitly accounted arena intentionally survives + # safe failures; no per-copy views/source tensors may survive. + self.assertTrue(all(ref() is None or ref() is self.controller._pinned_arena + for ref in allocated)) + expected_arena = (self.controller._pinned_arena.numel() + if self.controller._pinned_arena is not None else 0) + self.assertEqual(self.controller.stats()["pinned_arena_bytes"], expected_arena) + self.assertLessEqual(expected_arena, self.controller.stats()["pinned_capacity_bytes"]) + for key in ("output_bytes", "pinned_bytes", "device_bytes", "pending_copies"): + self.assertEqual(self.controller.stats()[key], 0) + self.controller.poll_completed() + self.controller.poll_drained() + + def test_unproven_event_drain_quarantines_without_budget_reuse(self): + cache = PrefixFeatureCache(schema(), 4, 128) + controller = CaptureController(cache, schema(), 128, 128, 128) + execution = identity(99) + tokens = tuple(range(8)) + lease = cache.lookup_and_pin("ns", tokens, 8) + reservation = controller.try_reserve_output(execution, 8, 0, 0) + plan = controller.freeze_plan(execution, "ns", tokens, 8, 0, lease, reservation) + class UnprovenEvent: + def synchronize(self): + raise RuntimeError("producer drain unavailable") + future = controller.submit(plan, CapturedSpan(execution, 0, 8, features(8), UnprovenEvent())) + with self.assertRaisesRegex(RuntimeError, "quarantined"): + future.result(3) + drain = controller.invalidate(execution, "cancel after failed event") + self.assertFalse(drain.wait(0.01)) + controller.release_output(reservation) + self.assertEqual(controller.stats()["device_bytes"], 128) + self.assertEqual(controller.stats()["quarantined_executions"], 1) + with self.assertRaisesRegex(RuntimeError, "not drained"): + controller.shutdown(wait=True) + + def test_completion_requires_exactly_one_outcome(self): + with self.assertRaises(CaptureContractError): + CaptureCompletion(identity()) + with self.assertRaises(CaptureContractError): + CaptureCompletion(identity(), sample=object(), error=RuntimeError()) + + +if __name__ == "__main__": + unittest.main() From a77aec71f616e7f693179a6aa68a32ccd0acd391 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 01:57:28 +0800 Subject: [PATCH 03/17] feat(capture): distinguish atomic capacity refusal from prompt failure Only the explicit begin/begin_batch capture_capacity_declined response with reservation_granted=false authorizes backpressure retry. Other errors remain fail-closed under the existing protocol. --- specforge/inference/capture_protocol.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/specforge/inference/capture_protocol.py b/specforge/inference/capture_protocol.py index c00cde146b..05d0d59d06 100644 --- a/specforge/inference/capture_protocol.py +++ b/specforge/inference/capture_protocol.py @@ -13,6 +13,10 @@ class CaptureControlRejected(RuntimeError): """The server explicitly rejected the operation without granting a permit.""" +class CaptureCapacityDeclined(CaptureControlRejected): + """An atomic begin was refused before granting any reservation.""" + + def request_digest(tokens, spec): value = {"tokens": list(tokens), "features": spec.get("features", {}), "passthrough": spec.get("passthrough", [])} @@ -47,6 +51,10 @@ def call(self, body): except Exception: pass # Keep the original rejection recognizable to its owner. raise + if (isinstance(result, dict) and body.get('action') in ('begin', 'begin_batch') + and result.get('error_code') == 'capture_capacity_declined' + and result.get('reservation_granted') is False): + raise CaptureCapacityDeclined(str(result.get('error', 'capture capacity exhausted'))) if not isinstance(result, dict) or result.get('error'): raise RuntimeError('capture control failed: ' + repr(result)) return result From 52c78a52b2820638fd878db05ff0b2071cec63b7 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 01:57:32 +0800 Subject: [PATCH 04/17] feat(runtime): integrate native HS attempt ownership and continuous rollout Select generic HS semantics from 0b3e00aa, keeping plugin-only capture and upstream durable ACK/receive pools. Explicit hs_capture/hs_prefix_cache negotiate immutable attempt ownership; coalesced begin retains independent completion and bounded byte credits. Atomic capacity refusal retries the same lease without prompt failures. Fenced failed reads quarantine storage and registration until owned transport shutdown. Retain generic full-record filtering and optimizer/scheduler/polling compatibility. Native managed capture and live gate keep official version checks enabled. Requires the separately migrated capture_protocol.py v1 plus CaptureCapacityDeclined from the plugin layer. Validation: 221 CPU tests, 219 pass / 1 skip / 1 expected failure; exact dirty source manifest in migration tests/p3-cpu-r2. No GPU numerical or production acceptance inferred. --- specforge/config/schema.py | 39 +- specforge/data/prompt_builder.py | 67 +- .../inference/adapters/server_capture.py | 199 ++++- specforge/inference/rollout_worker.py | 28 +- specforge/launch.py | 350 +++++++- specforge/launch_plan.py | 20 +- specforge/lr_scheduler.py | 35 + specforge/optimizer.py | 10 +- specforge/runtime/control_plane/controller.py | 8 +- .../runtime/data_plane/feature_dataloader.py | 4 +- specforge/runtime/data_plane/feature_store.py | 9 + .../runtime/data_plane/mooncake_store.py | 761 +++++++++++++++++- .../runtime/data_plane/ref_distributor.py | 4 +- .../data_plane/streaming_ref_channel.py | 4 + specforge/training/assembly.py | 4 + specforge/training/disaggregated.py | 46 +- tests/plana/test_runtime_polling.py | 169 ++++ tests/test_data/test_prompt_length_filter.py | 121 +++ tests/test_optimizer/test_torchspec_compat.py | 42 + .../test_capture_begin_parallel.py | 202 +++++ tests/test_runtime/test_capture_fencing.py | 253 ++++++ .../test_capture_producer_coalescing.py | 513 ++++++++++++ tests/test_runtime/test_capture_transfer.py | 256 ++++++ tests/test_runtime/test_native_hs_pipeline.py | 116 +++ .../test_online_terminal_cleanup.py | 226 ++++++ .../test_runtime/test_server_capture_gate.py | 2 +- 26 files changed, 3402 insertions(+), 86 deletions(-) create mode 100644 tests/plana/test_runtime_polling.py create mode 100644 tests/test_data/test_prompt_length_filter.py create mode 100644 tests/test_optimizer/test_torchspec_compat.py create mode 100644 tests/test_runtime/test_capture_begin_parallel.py create mode 100644 tests/test_runtime/test_capture_fencing.py create mode 100644 tests/test_runtime/test_capture_producer_coalescing.py create mode 100644 tests/test_runtime/test_capture_transfer.py create mode 100644 tests/test_runtime/test_native_hs_pipeline.py create mode 100644 tests/test_runtime/test_online_terminal_cleanup.py diff --git a/specforge/config/schema.py b/specforge/config/schema.py index fdf2332919..4a3bc8d026 100644 --- a/specforge/config/schema.py +++ b/specforge/config/schema.py @@ -141,7 +141,7 @@ #: ``sglang_*`` fields that carry raw argv tokens instead of one flag. SGLANG_PASSTHROUGH_FIELDS = frozenset({"sglang_extra_args"}) #: Capture-server environment the managed-local launcher sets itself. -_OWNED_SERVER_ENV_PREFIXES = ("MOONCAKE_", "DISAGG_") +_OWNED_SERVER_ENV_PREFIXES = ("MOONCAKE_", "DISAGG_", "SPECFORGE_HS_") _OWNED_SERVER_ENV = { "CUDA_VISIBLE_DEVICES": "capture_servers[].cuda_visible_devices", "ASCEND_RT_VISIBLE_DEVICES": "capture_servers[].cuda_visible_devices", @@ -419,6 +419,7 @@ class DataConfig(StrictConfigModel): #: offline evaluation — directory of precomputed hidden-state .ckpt files. eval_hidden_states_path: str = "" max_length: int = Field(default=2048, gt=0) + overlength_policy: Literal["truncate", "drop"] = "truncate" chat_template: str = "llama3" is_preformatted: bool = False train_only_last_turn: bool = False @@ -432,6 +433,8 @@ class DataConfig(StrictConfigModel): @model_validator(mode="after") def _exactly_one_source(self): + if self.overlength_policy == "drop" and not self.prompts_path: + raise ValueError("data.overlength_policy=drop requires native data.prompts_path") sources = [ bool(self.train_data_path), bool(self.prompts_path), @@ -495,6 +498,9 @@ class RuntimeConfig(StrictConfigModel): """Streaming bounds shared by unified disaggregated producer roles.""" producer_lease: int = Field(default=8, gt=0) + producer_backpressure_poll_s: float = Field(default=0.2, gt=0, allow_inf_nan=False) + consumer_ref_poll_s: float = Field(default=0.1, gt=0, allow_inf_nan=False) + ref_distributor_poll_s: float = Field(default=0.05, gt=0, allow_inf_nan=False) consumer_dispatch: Literal[ "round_robin", "cost_balanced", @@ -509,6 +515,14 @@ class RuntimeConfig(StrictConfigModel): feature_store_max_resident_bytes: Optional[int] = Field(default=None, gt=0) feature_store_max_quarantined_bytes: int = Field(default=8 << 30, ge=0) + @field_validator("producer_backpressure_poll_s", "consumer_ref_poll_s", + "ref_distributor_poll_s", mode="before") + @classmethod + def _poll_seconds_numeric(cls, value): + if isinstance(value, bool): + raise ValueError("polling intervals must be positive finite numbers") + return value + @model_validator(mode="after") def _validate_watermarks(self): resolve_ref_watermarks( @@ -841,6 +855,17 @@ class DisaggregatedDeploymentConfig(StrictConfigModel): #: backend: managed_local probes each capture server's interpreter while #: planning; the producer reads an external server's ``/server_info``. server_capture: Literal["auto", "plugin"] = "auto" + #: Native HS protocol and content-cache reuse are separate opt-ins so + #: cache-off/on comparisons share the same publication/ownership path. + hs_capture: bool = False + hs_prefix_cache: bool = False + hs_chunk_tokens: int = Field(default=256, gt=0) + hs_cache_bytes: int = Field(default=1 << 30, gt=0) + hs_pinned_bytes: int = Field(default=256 << 20, gt=0) + hs_output_bytes: int = Field(default=512 << 20, gt=0) + hs_device_bytes: int = Field(default=256 << 20, gt=0) + hs_max_workers: int = Field(default=4, gt=0) + hs_attempt_capacity: int = Field(default=4096, gt=0) mooncake_metadata_server: Optional[str] = None mooncake_master_server_addr: Optional[str] = None mooncake_local_hostname: Optional[str] = None @@ -880,6 +905,10 @@ class DisaggregatedDeploymentConfig(StrictConfigModel): @model_validator(mode="after") def _validate_store(self): + if self.hs_prefix_cache and not self.hs_capture: + raise ValueError("hs_prefix_cache requires hs_capture") + if self.hs_capture and self.backend != "mooncake": + raise ValueError("HS capture requires backend=mooncake") if not self.control_dir: raise ValueError("deployment.disaggregated.control_dir must not be empty") if self.managed_local is not None: @@ -982,7 +1011,9 @@ class TrainingConfig(StrictConfigModel): #: Joint logits loss; independent of objective checkpointing. fused_logits_loss: bool = True learning_rate: float = Field(default=1e-4, gt=0.0) - lr_scheduler: Literal["cosine", "constant"] = "cosine" + lr_scheduler: Literal["cosine", "constant", "torchspec_cosine"] = "cosine" + weight_decay: float = Field(default=0.0, ge=0.0) + min_lr: float = Field(default=0.0, ge=0.0) warmup_ratio: float = Field(default=0.015, ge=0.0, le=1.0) max_grad_norm: float = Field(default=0.5, gt=0.0) #: Keep FP32 Adam masters and moments on CPU while the trainable draft @@ -1074,6 +1105,10 @@ class TrainingConfig(StrictConfigModel): @model_validator(mode="after") def _validate_training_shape(self): + if self.min_lr > self.learning_rate: + raise ValueError("training.min_lr must not exceed training.learning_rate") + if self.min_lr and self.lr_scheduler == "constant": + raise ValueError("training.min_lr requires a cosine scheduler") if not 0.0 <= self.dpace_alpha <= 1.0: raise ValueError("training.dpace_alpha must be in [0, 1]") if not 0.0 < self.down_sample_ratio <= 1.0: diff --git a/specforge/data/prompt_builder.py b/specforge/data/prompt_builder.py index 30a7c5c9f2..3fd84161b2 100644 --- a/specforge/data/prompt_builder.py +++ b/specforge/data/prompt_builder.py @@ -32,6 +32,8 @@ def prepare_prompt_tasks( min_loss_tokens: int = 1, max_prompts: int | None = None, loss_mask_filter: Callable[[Sequence[int]], bool] | None = None, + overlength_policy: str = "truncate", + filter_report: dict[str, Any] | None = None, ) -> Sequence[PromptTaskDict]: """Prepare runtime prompt dictionaries from a JSONL file. @@ -43,11 +45,17 @@ def prepare_prompt_tasks( lazy random-access sequence so large Arrow datasets are not expanded into Python token lists before rollout starts. - ``max_prompts`` caps input records before preprocessing and filtering on - both paths. Rejected records count toward the cap and are not replaced, - so fewer prompts may be returned. ``None`` and ``0`` mean no cap. + Native ``overlength_policy="drop"`` rejects full P+G records longer than + ``max_length`` before any slicing. Equal-length records remain unchanged. + In this mode, ``max_prompts`` caps accepted prompts for compatibility with + existing filtered datasets. Raw conversations and native ``truncate`` cap + input records before preprocessing and filtering; rejected rows consume + that quota. ``None`` and ``0`` mean no cap in either mode. + ``filter_report`` optionally receives the filter counts and dropped IDs. """ + if overlength_policy not in {"truncate", "drop"}: + raise ValueError("overlength_policy must be truncate or drop") _validate_options( max_length=max_length, min_loss_tokens=min_loss_tokens, @@ -77,15 +85,22 @@ def prepare_prompt_tasks( (record, f"{path_string}:{line_number}") for line_number, record in _iter_records(path_string) ) - if limit is not None: + if limit is not None and overlength_policy == "truncate": rows = islice(rows, limit) return _materialize_prompt_tasks( rows, max_length=max_length, min_loss_tokens=min_loss_tokens, loss_mask_filter=loss_mask_filter, + accepted_limit=limit if overlength_policy == "drop" else None, + overlength_policy=overlength_policy, + filter_report=filter_report, ) + if overlength_policy == "drop": + raise ValueError( + "overlength_policy=drop requires pre-tokenized input_ids/loss_mask" + ) return _prepare_raw_prompts( path_string, tokenizer, @@ -195,22 +210,51 @@ def _materialize_prompt_tasks( max_length: int, min_loss_tokens: int, loss_mask_filter: Callable[[Sequence[int]], bool] | None, + accepted_limit: int | None = None, + overlength_policy: str = "truncate", + filter_report: dict[str, Any] | None = None, ) -> list[PromptTaskDict]: + report = filter_report if filter_report is not None else {} + report.update( + policy=overlength_policy, + max_length=max_length, + rows_seen=0, + accepted=0, + dropped_overlength=0, + dropped_other=0, + kept_input_tokens=0, + kept_loss_tokens=0, + dropped_samples=[], + ) prompts: list[PromptTaskDict] = [] for record, source in rows: + report["rows_seen"] += 1 + before_overlength = report["dropped_overlength"] prompt = _prompt_from_record( record, source=source, max_length=max_length, min_loss_tokens=min_loss_tokens, + overlength_policy=overlength_policy, + filter_report=report, ) if prompt is None: + if report["dropped_overlength"] == before_overlength: + report["dropped_other"] += 1 continue if loss_mask_filter is not None and not loss_mask_filter( prompt["payload"]["loss_mask"] ): + report["dropped_other"] += 1 continue prompts.append(prompt) + report["accepted"] += 1 + report["kept_input_tokens"] += len(prompt["payload"]["input_ids"]) + report["kept_loss_tokens"] += sum(prompt["payload"]["loss_mask"]) + if accepted_limit is not None and len(prompts) >= accepted_limit: + break + if overlength_policy == "drop": + print("[prompt-filter] " + json.dumps(report, ensure_ascii=False), flush=True) return prompts @@ -220,6 +264,8 @@ def _prompt_from_record( source: str, max_length: int, min_loss_tokens: int, + overlength_policy: str = "truncate", + filter_report: dict[str, Any] | None = None, ) -> PromptTaskDict | None: if "input_ids" not in record or "loss_mask" not in record: raise ValueError(f"{source} must contain both input_ids and loss_mask") @@ -238,6 +284,19 @@ def _prompt_from_record( if not input_ids: raise ValueError(f"{source} contains an empty token sequence") + if overlength_policy == "drop" and len(input_ids) > max_length: + if filter_report is not None: + filter_report["dropped_overlength"] += 1 + filter_report["dropped_samples"].append( + { + "id": record.get("id"), + "source": source, + "length": len(input_ids), + "loss_tokens": sum(loss_mask), + } + ) + return None + input_ids = input_ids[:max_length] loss_mask = loss_mask[:max_length] if sum(loss_mask) < min_loss_tokens: diff --git a/specforge/inference/adapters/server_capture.py b/specforge/inference/adapters/server_capture.py index 7126a3f294..d233eb17bf 100644 --- a/specforge/inference/adapters/server_capture.py +++ b/specforge/inference/adapters/server_capture.py @@ -27,6 +27,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import json import logging import threading @@ -200,6 +202,9 @@ def __init__( target_model_version: str = "unknown", server_capture: str = "plugin", server_info_fn: Optional[Callable[[str, float], Mapping[str, Any]]] = None, + hs_capture: bool = False, + hs_prefix_cache: bool = False, + control_fn: Optional[Callable[..., Any]] = None, ) -> None: if server_capture not in ("auto", *SERVER_CAPTURE_BACKENDS): raise ValueError( @@ -238,6 +243,16 @@ def __init__( self.request_input_adapter = request_input_adapter self.timeout_s = timeout_s self.post_fn = post_fn or _SessionPost() + if hs_prefix_cache and not hs_capture: + raise ValueError("hs_prefix_cache requires explicit hs_capture opt-in") + self.hs_capture = hs_capture + self.hs_prefix_cache = hs_prefix_cache + from specforge.inference.capture_protocol import CaptureProtocol + + self._capture_protocol = ( + CaptureProtocol(base_url, control_fn or self.post_fn, timeout_s) + if hs_capture else None + ) self.target_model_version = target_model_version #: ``auto`` until the first request reads the server's backend. self.server_capture = server_capture @@ -382,7 +397,7 @@ def _ref_from_result( run_id=self.run_id, source_task_id=task.task_id, feature_store_uri=f"mooncake://{result['store_id']}/{sample_id}", - feature_keys={n: f"{sample_id}/{n}" for n in specs}, + feature_keys=(dict(result["physical_keys"]) if result.get("publication_ownership") == "attempt-v1" else {n: f"{sample_id}/{n}" for n in specs}), feature_specs=specs, strategy=self.strategy, schema_version=SCHEMA_VERSION, @@ -408,6 +423,15 @@ def _ref_from_result( ), }, ) + if result.get("protocol_version") == 1: + ref.metadata["capture_fence"] = { + **{name: result[name] for name in ("server_epoch", "store_id", "sample_id", "gen", "attempt_id", "reservation_token")}, + "protocol_version": 1, "control_url": self.base_url + "/spec_capture/control", + "timeout_s": self.timeout_s, + **{name: result[name] for name in ('object_key_layout', 'publication_ownership', 'physical_keys', 'writer_drained', 'state') if name in result}, + } + ref.metadata["capture_diagnostics"] = {name: result.get(name) for name in + ("H", "L", "N", "span", "execution_epoch", "rank", "reused_tokens", "capture_namespace", "capture_signature", "timings")} if "arena" in result: # Arena objects stay on the capture server; currently only online # NVLink capture uses the arena. @@ -433,16 +457,136 @@ def produce_refs( transport-level error raises — the worker fails the whole lease batch retryable, mirroring ``generate_features`` semantics. """ + prepared = self.prepare_refs_batches([tasks], capture=capture) + self.register_prepared_batches(prepared) + return self.produce_prepared(prepared[0], capture=capture) + + def prepare_refs_batches(self, task_batches, *, capture: CaptureConfig): + """Build bounded lease groups without registering a writer yet.""" + self._resolve_server_capture() + capability = self._capture_protocol.capability() if self._capture_protocol is not None else None + if self.hs_capture: + if capability is None or capability.get("published_ownership") != "attempt-v1": + raise RuntimeError("HS capture requires native plugin attempt-v1 capability") + if capability.get("cache_enabled") is not self.hs_prefix_cache: + raise RuntimeError("HS cache setting disagrees with server capability") + if capability is not None: + required = ("prepare_capture_retry", "track_capture_fence") + if any(not callable(getattr(self.store, name, None)) for name in required): + raise TypeError("negotiated capture requires attempt-fenced store") + requested_layers = tuple(capture.aux_hidden_state_layer_ids or ()) + schema = {item["name"]: item for item in capability["schema"]} + if requested_layers and schema.get("aux", {}).get("width") != capture.expected_aux_width: + raise CaptureMismatchError("server capability aux width mismatch") + if self.schema.last_hidden_feature is not None and schema.get("last_hidden", {}).get("width") != capture.target_hidden_size: + raise CaptureMismatchError("server capability last hidden width mismatch") + if requested_layers and tuple(schema.get("aux", {}).get("layer_ids", ())) != requested_layers: + raise CaptureMismatchError("server capability aux layer order mismatch") + prepared = [] + for tasks in task_batches: + body = self._request_inputs(tasks) + body["extra_key"] = [uuid.uuid4().hex for _ in tasks] + payloads = [self._spec_capture_payload(task) for task in tasks] + if capability is not None: + for payload in payloads: + self.store.prepare_capture_retry(payload["sample_id"]) + if capability["cache_enabled"]: + body["extra_key"] = [capability["capture_namespace"]] * len(tasks) + prepared.append(dict(tasks=tasks, body=body, payloads=payloads, + capability=capability, + estimated_bytes=self._output_bytes(tasks, capability))) + return prepared + + def _output_bytes(self, tasks, capability): + # The negotiated schema fixes the complete engine tensor width/dtype. + # Passthrough tensors are int64 and have client-declared trailing dims. + if capability is None: + return None + schema = {item["name"]: item for item in capability["schema"]} + per_token = 0 + for artifact, feature in (("aux", self.schema.aux_feature), + ("last_hidden", self.schema.last_hidden_feature)): + if feature is not None: + item = schema[artifact] + width = int(item["width"]) + if width < 1 or item["dtype"] not in _DTYPE_BYTES: + raise ValueError("invalid negotiated capture output schema") + per_token += width * _DTYPE_BYTES[item["dtype"]] + for _name, _key, trailing in self.schema.passthrough: + per_token += _spec_nbytes((1, *trailing), "int64") + if self.schema.attention_mask_feature is not None: + per_token += 8 + return sum(len(task.payload["input_ids"]) * per_token for task in tasks) + + def register_prepared_batches(self, prepared): + """Coalesce all currently ready leases into one endpoint begin_batch.""" + if not prepared: + return + capability = prepared[0]["capability"] + if any(item["capability"] != capability for item in prepared): + raise RuntimeError("capture capability changed within registration batch") + if capability is None: + return + payloads = [payload for item in prepared for payload in item["payloads"]] + tasks = [task for item in prepared for task in item["tasks"]] + requests = [] + for item in prepared: + item["begin_control_url"] = self._capture_protocol.url + for task, payload in zip(tasks, payloads): + names = [*payload["features"].values(), *(x["name"] for x in payload.get("passthrough", ()))] + # Freeze the begin identity before crossing HTTP. Even a lost reply + # must not let a late cleanup retarget a peer's replacement writer. + if "attempt_id" in payload and not all(item.get("registration_refused") for item in prepared): + raise RuntimeError("prepared capture registration cannot be replayed") + payload.update(attempt_id=payload.get("attempt_id", uuid.uuid4().hex), + server_epoch=capability["server_epoch"], replace=False) + requests.append((payload, task.payload["input_ids"], payload["attempt_id"], names)) + from specforge.inference.capture_protocol import CaptureCapacityDeclined + try: + fences = self.store.begin_capture_attempts(self._capture_protocol, capability, requests) + except CaptureCapacityDeclined: + for item in prepared: + item["registration_refused"] = True + raise + for item in prepared: + item["registration_refused"] = False + for payload, fence in zip(payloads, fences): + payload.update({name: capability[name] for name in ("server_epoch", "capture_namespace", "capture_signature")}) + payload.update({name: fence[name] for name in ("protocol_version", "attempt_id", "reservation_token")}) + if fence.get("object_key_layout") == "attempt-v1": + payload["object_key_layout"] = "attempt-v1" + + def unreturned_prepared(self, prepared, refs): + """Retain only per-task failures for exact-attempt retirement.""" + returned = {ref.sample_id for ref in refs} + pairs = [(task, payload) for task, payload in zip(prepared["tasks"], prepared["payloads"]) + if payload["sample_id"] not in returned] + if not pairs: + return None + tasks, payloads = map(list, zip(*pairs)) + return {**prepared, "tasks": tasks, "payloads": payloads, + "estimated_bytes": self._output_bytes(tasks, prepared["capability"])} + + def discard_prepared_batches(self, prepared): + # Resolve uncertain begin replies using the store's immutable original + # intent, then retire that exact writer before releasing byte credits. + for item in prepared: + for payload in item["payloads"]: + if "attempt_id" not in payload: + continue # This lease never registered a server writer. + self.store.discard_capture_attempt( + payload["sample_id"], server_epoch=payload["server_epoch"], + attempt_id=payload["attempt_id"], control_url=item["begin_control_url"], + reservation_token=payload.get("reservation_token"), + ) + + def produce_prepared(self, prepared, *, capture: CaptureConfig): + tasks = prepared["tasks"] + body = prepared["body"] + capture_payloads = prepared["payloads"] + capability = prepared["capability"] server_capture = self._resolve_server_capture() - body = self._request_inputs(tasks) - # A fresh cache namespace forces a full prefill on every attempt while - # leaving SGLang free to use the radix data structure required by hybrid - # targets. Retries need a new key because the prior attempt may have - # populated its namespace before the response was lost. - body["extra_key"] = [uuid.uuid4().hex for _ in tasks] sampling_params = {"temperature": 0.0, "max_new_tokens": 0} - capture_payloads = [self._spec_capture_payload(t) for t in tasks] - # custom_params values are JSON scalars: one encoded spec per request. body["sampling_params"] = [ { **sampling_params, @@ -463,9 +607,14 @@ def produce_refs( feature_names=feature_names, server_url=self.base_url, ) - rows = self.post_fn( - f"{self.base_url}/generate", json_body=body, timeout=self.timeout_s - ) + try: + rows = self.post_fn( + f"{self.base_url}/generate", json_body=body, timeout=self.timeout_s + ) + except Exception as exc: + if self._capture_protocol is not None and getattr(getattr(exc, 'response', None), 'status_code', None) == 409: + self._capture_protocol.capability(refresh=True) + raise rows = _flatten_list_wrappers(rows) if len(rows) != len(tasks): raise RuntimeError( @@ -475,7 +624,7 @@ def produce_refs( capture_hint = _CAPTURE_HINTS[server_capture] out: List[Union[Any, ServerCaptureFailure]] = [] successful_refs = [] - for task, row in zip(tasks, rows): + for task, row, payload in zip(tasks, rows, capture_payloads): if not isinstance(row, dict): raise RuntimeError( "spec-capture server returned a non-object row for task " @@ -526,7 +675,27 @@ def produce_refs( f"task {task.task_id}: {actual_identity} != " f"{expected_identity}" ) + if capability is not None: + for name in ("protocol_version", "server_epoch", "capture_namespace", "capture_signature", "attempt_id", "reservation_token"): + if result.get(name) != payload.get(name): + if name == 'server_epoch': + self._capture_protocol.invalidate_epoch(result.get(name)) + raise RuntimeError("negotiated capture response identity mismatch: " + name) + if capability.get('published_ownership') == 'attempt-v1': + if result.get('publication_ownership') != 'attempt-v1' or result.get('state') != 'PUBLISHED' or not result.get('writer_drained'): + raise RuntimeError('capture response omitted drained ownership transfer') + from specforge.inference.capture_protocol import attempt_keys + names = [*payload['features'].values(), *(x['name'] for x in payload.get('passthrough', ()))] + keys = attempt_keys(result, names) + if result.get('physical_keys') != keys or any(result.get('features', {}).get(name, {}).get('key') != key for name, key in keys.items()): + raise RuntimeError('capture response immutable keys mismatch') + if self.hs_capture: + N, L = len(task.payload["input_ids"]), result.get("L") + if not isinstance(L, int) or not 0 <= L < N or result.get("N") != N or result.get("span") != [L, N]: + raise RuntimeError("negotiated capture returned invalid absolute span") ref = self._ref_from_result(task, result, capture) + if capability is not None and ref.estimated_bytes > self._output_bytes([task], capability): + raise RuntimeError("capture output exceeded negotiated byte reservation") # A capture shorter than the prompt is corrupt: it means cache # isolation or another full-prefill invariant failed despite the # request's fresh namespace. @@ -538,7 +707,7 @@ def produce_refs( } if short: self.store.adopt(ref) - self.store.abort(ref.sample_id, reason="seq-len-mismatch") + abort_feature_ref(self.store, ref, reason="seq-len-mismatch") out.append( ServerCaptureFailure( task_id=task.task_id, @@ -578,7 +747,7 @@ def produce_refs( # Loud boundary failure; free the server-written keys so a # mismatched sample is never consumable. self.store.adopt(ref) - self.store.abort(ref.sample_id, reason=f"contract:{exc}") + abort_feature_ref(self.store, ref, reason=f"contract:{exc}") out.append( ServerCaptureFailure( task_id=task.task_id, reason=str(exc), retryable=False diff --git a/specforge/inference/rollout_worker.py b/specforge/inference/rollout_worker.py index eebf2a8035..d13e5ac8fa 100644 --- a/specforge/inference/rollout_worker.py +++ b/specforge/inference/rollout_worker.py @@ -123,14 +123,34 @@ def _record_commit(self, count: int) -> None: def _sample_id(self, task: PromptTask) -> str: return f"{self.run_id}:{task.task_id}" - def run_once(self, max_tasks: int) -> List[SampleRef]: + def lease_tasks(self, max_tasks: int) -> List[PromptTask]: + """Lease once, including queued preparation in worker health.""" with self._health_lock: if self._state in ("stopped", "draining"): return [] tasks = self.controller.lease_prompt_tasks(self.worker_id, max_tasks) + if tasks: + self._begin_batch(len(tasks)) + return tasks + + def fail_leased(self, tasks, reason: str, *, unhealthy: bool = True) -> None: + if unhealthy: + self._mark_unhealthy(reason) + self.controller.fail_prompt_tasks( + self.worker_id, [task.task_id for task in tasks], + reason=reason, retryable=True, + ) + self._finish_batch(len(tasks)) + + def run_once(self, max_tasks: int) -> List[SampleRef]: + return self.run_leased(self.lease_tasks(max_tasks), max_tasks=max_tasks) + + def run_leased(self, tasks: List[PromptTask], *, max_tasks: Optional[int] = None, + prepared=None) -> List[SampleRef]: + """Execute an existing lease; preparation never leases a second time.""" if not tasks: return [] - self._begin_batch(len(tasks)) + max_tasks = len(tasks) if max_tasks is None else max_tasks # Legacy colocated target-TP training used a drop-last target batch. # A partial target batch would give ranks different local batch sizes # and break the all-rank draft/FSDP step contract. @@ -151,6 +171,10 @@ def run_once(self, max_tasks: int) -> List[SampleRef]: "target-TP batch partitioning is supported for colocated " "feature capture only" ) + if prepared is not None: + produce_refs = lambda _tasks, *, capture: self.feature_source.produce_prepared( + prepared, capture=capture + ) return self._run_ref_source(tasks, produce_refs) try: diff --git a/specforge/launch.py b/specforge/launch.py index 3a0b19c881..3cddfcd8c7 100644 --- a/specforge/launch.py +++ b/specforge/launch.py @@ -10,6 +10,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import logging import os from typing import Any, Callable, List, Mapping, Optional, Tuple @@ -386,8 +388,8 @@ def _publish_refs_with_cleanup( cleanup_errors = [] for ref in transaction.unpublished_refs: try: - feature_store.abort( - ref.sample_id, + abort_feature_ref( + feature_store, ref, reason="producer-ref-publication-failed", ) except Exception as cleanup_exc: @@ -402,6 +404,11 @@ def _publish_refs_with_cleanup( f"{cleanup_errors}" ) from publish_exc raise + finally: + relinquish = getattr(feature_store, "relinquish_published", None) + if callable(relinquish): + for ref in transaction.published_refs: + relinquish(ref) def _epoch_online_prompts( @@ -838,8 +845,12 @@ def build_disagg_online_producer( server-capture transport; a *sequence* of sources fans out to one logical worker per source (the multi-server topology). Each logical worker keeps ``producer_concurrency`` capture calls in flight and replenishes a slot as - soon as that call completes. There is intentionally no in-process - target-model path. + soon as that call completes. Negotiated capture adapters register the + currently ready leases together per endpoint, then generate and publish + each lease independently. Prepared leases occupy bounded capture slots; + negotiated output byte reservations include all unfinished server writes + in the global feature-store hard cap until publication or exact cleanup. + There is intentionally no in-process target-model path. The producer has no training ledger or local ref queue; the consumer owns deduplication and durable acknowledgements. Returns ``(workers, drive_producer)``: @@ -1081,10 +1092,12 @@ def drive_producer(max_rounds: int = 1_000_000, should_stop=None) -> int: "accounted_consumed": channel.consumed_remote(), "resident_bytes": 0, "published_bytes": 0, + "reserved_bytes": 0, } published_sizes = deque() last_publish_log = {"t": time.perf_counter()} dead: dict = {} # worker_id -> last failure reason + inactive: dict = {} # explicitly stopped/draining workers # Set once the feeder will ingest nothing more (plan exhausted, stop, # or failure); only then may an empty pool end a worker. ingest_closed = threading.Event() @@ -1144,10 +1157,11 @@ def resident_bytes() -> int: with publish_lock: return reconcile_consumed_locked() - def publish_refs(w, refs, run_once_start: float) -> None: - if not refs: - return + def publish_refs(w, refs, run_once_start: float, reservation: int = 0) -> None: with publish_lock: + if not refs: + state["reserved_bytes"] -= reservation + return current_bytes = reconcile_consumed_locked() ref_sizes = [max(0, int(ref.estimated_bytes or 0)) for ref in refs] projected_bytes = current_bytes + sum(ref_sizes) @@ -1158,8 +1172,8 @@ def publish_refs(w, refs, run_once_start: float) -> None: cleanup_errors = [] for ref in refs: try: - feature_store.abort( - ref.sample_id, + abort_feature_ref( + feature_store, ref, reason="producer-resident-byte-hard-cap", ) except Exception as exc: @@ -1174,11 +1188,18 @@ def publish_refs(w, refs, run_once_start: float) -> None: if cleanup_errors: message += f"; cleanup errors={cleanup_errors}" raise MemoryError(message) - _publish_refs_with_cleanup( - channel=channel, - feature_store=feature_store, - refs=refs, - ) + try: + _publish_refs_with_cleanup( + channel=channel, + feature_store=feature_store, + refs=refs, + ) + except BaseException: + # A possibly visible prefix still owns its original credit. + # Stop peer admissions before releasing the shared lock. + abort.set() + raise + state["reserved_bytes"] -= reservation if channel.consumed_state_remote()[1] is None: published_sizes.extend(ref_sizes) state["published_bytes"] += sum(ref_sizes) @@ -1214,8 +1235,8 @@ def abort_unpublished(futures) -> None: continue for ref in refs: try: - feature_store.abort( - ref.sample_id, + abort_feature_ref( + feature_store, ref, reason="producer-driver-failed-before-publication", ) except Exception: @@ -1223,7 +1244,7 @@ def abort_unpublished(futures) -> None: "failed to abort unpublished ref %s", ref.sample_id ) - def run_worker(w) -> None: + def run_worker_legacy(w) -> None: failures = 0 submitted = 0 accepting = True @@ -1361,6 +1382,291 @@ def run_worker(w) -> None: abort_unpublished(futures) raise + def run_worker(w) -> None: + source = w.feature_source + if getattr(source, "_capture_protocol", None) is None: + return run_worker_legacy(w) + + # One endpoint's network registration has its own executor. The + # coordinator only leases, reserves credits and harvests futures; + # an in-flight begin never blocks independent ref publication. + records = [] + futures = {} + preparing = registering = cleaning = None + failures = submitted = 0 + accepting = True + backpressure_started = None + capacity_blocked = False + next_registration_at = 0.0 + from specforge.inference.capture_protocol import CaptureCapacityDeclined + + def release_credit(record): + with publish_lock: + state["reserved_bytes"] -= record["credit"] + record["credit"] = 0 + + def reserve(record): + if record["credit"]: + return True # Same leased attempt retains its original credit. + size = record["prepared"]["estimated_bytes"] + if size is None or feature_store_max_resident_bytes is None: + return True + with publish_lock: + if abort.is_set(): + return False + current = reconcile_consumed_locked() + if size > feature_store_max_resident_bytes: + raise MemoryError( + "producer feature-store hard cap exceeded: " + f"sample reservation={size} > limit={feature_store_max_resident_bytes} bytes" + ) + if current + state["reserved_bytes"] + size > feature_store_max_resident_bytes: + return False + state["reserved_bytes"] += size + record["credit"] = size + return True + + def failed(exc): + nonlocal failures, accepting + failures += 1 + if failures >= max_worker_failures: + accepting = False + dead[w.worker_id] = str(exc) + + def clean_batch(group): + source.discard_prepared_batches([r["prepared"] for r in group]) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=producer_concurrency, + thread_name_prefix=f"capture-{w.worker_id}", + ) as executor, concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix=f"prepare-{w.worker_id}", + ) as preparation, concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix=f"register-{w.worker_id}", + ) as registration: + try: + while True: + health_state = w.health()["state"] + if health_state in ("stopped", "draining"): + accepting = False + inactive[w.worker_id] = health_state + if abort.is_set() or (should_stop is not None and should_stop()): + accepting = False + # Harvest generates first, including while a different + # registration or uncertain-attempt cleanup is blocked. + for future in list(futures): + if not future.done(): + continue + record = futures.pop(future) + # A cancelled queued future never entered run_leased. + record["settled"] = not future.cancelled() + try: + refs = future.result() + except Exception as exc: + record.update(stage="cleanup", error=exc) + failed(exc) + else: + failures = 0 + credit = record["credit"] + missing = source.unreturned_prepared(record["prepared"], refs) + if missing is not None: + # Per-task errors settle controller leases but + # can still own server reservations or writes. + # Publish successes now; retain only failed + # identities and their credits until drained. + failed_credit = missing["estimated_bytes"] if credit else 0 + record.update(prepared=missing, tasks=missing["tasks"], + credit=failed_credit, stage="cleanup") + credit -= failed_credit + else: + record["credit"] = 0 + records.remove(record) + publish_refs(w, refs, record["start"], credit) + + if cleaning is not None and cleaning[0].done(): + future, group = cleaning + future.result() # Failure retains original ownership and aborts the driver. + for record in group: + release_credit(record) + if not record["settled"]: + w.fail_leased(record["tasks"], str(record["error"]), + unhealthy=not record.get("cancelled", False)) + records.remove(record) + cleaning = None + + if registering is not None and registering[0].done(): + future, group = registering + registering = None + try: + future.result() + except CaptureCapacityDeclined: + capacity_blocked = True + next_registration_at = time.monotonic() + backpressure_poll_s + for record in group: + record["stage"] = "pending" + except Exception as exc: + failed(exc) + for record in group: + record.update(stage="cleanup", error=exc) + else: + capacity_blocked = False + for record in group: + if (not accepting or abort.is_set() + or w.health()["state"] in ("stopped", "draining") + or (should_stop is not None and should_stop())): + accepting = False + record.update(stage="cleanup", error="producer-stopped", cancelled=True) + continue + record["stage"] = "generate" + future = executor.submit( + w.run_leased, record["tasks"], + max_tasks=worker_lease, prepared=record["prepared"], + ) + futures[future] = record + + if preparing is not None and preparing[0].done(): + future, group = preparing + preparing = None + try: + prepared = future.result() + if len(prepared) != len(group): + raise RuntimeError( + "capture preparation returned " + f"{len(prepared)} groups for {len(group)} leases" + ) + except Exception as exc: + failed(exc) + for record in group: + w.fail_leased(record["tasks"], f"prepare_refs:{exc}") + records.remove(record) + else: + for record, item in zip(group, prepared): + record.update(stage="pending", prepared=item) + + if not accepting: + # An unregistered prepared lease may be waiting for + # consumer credit. Cooperative stop must not depend + # on another ACK arriving before it can retire. + for record in records: + if record["stage"] == "pending": + record.update(stage="cleanup", error="producer-stopped", cancelled=True) + + # Credits for unfinished or uncertain server writes stay + # reserved until the exact attempt is physically drained. + if cleaning is None and registering is None: + group = [r for r in records if r["stage"] == "cleanup"] + if group: + cleaning = (registration.submit(clean_batch, group), group) + + in_flight = channel.in_flight_remote() + current_bytes = resident_bytes() + paused = in_flight >= consumer_quantum and flow_control.should_pause( + in_flight_refs=in_flight, resident_bytes=current_bytes, + ) + if accepting and not paused and submitted < round_budget and preparing is None: + group = [] + while len(records) < producer_concurrency and submitted < round_budget: + tasks = w.lease_tasks(worker_lease) + if not tasks: + break + record = dict(tasks=tasks, start=time.perf_counter(), + stage="prepare", credit=0, settled=False) + records.append(record) + group.append(record) + submitted += 1 + if group: + preparing = (preparation.submit( + source.prepare_refs_batches, + [r["tasks"] for r in group], capture=w.capture, + ), group) + + # Take the entire ready snapshot. Initial idle slots + # deterministically share one begin; late arrivals queue + # behind an active registration and coalesce next turn. + budget_wait = False + if (registering is None and cleaning is None + and time.monotonic() >= next_registration_at): + group = [] + for record in records: + if record["stage"] != "pending": + continue + if reserve(record): + record["stage"] = "register" + group.append(record) + else: + budget_wait = True + if group: + registering = (registration.submit( + source.register_prepared_batches, + [r["prepared"] for r in group], + ), group) + + if paused or budget_wait or capacity_blocked: + if backpressure_started is None: + backpressure_started = time.monotonic() + if (peer_wait_timeout_s is not None and + time.monotonic() - backpressure_started > peer_wait_timeout_s): + raise TimeoutError("producer backpressure timed out waiting for consumer progress") + else: + backpressure_started = None + + if not records: + if ingest_closed.is_set() and pool_drained(): + return + if not accepting or submitted >= round_budget: + return + waits = list(futures) + waits.extend(item[0] for item in (preparing, registering, cleaning) if item is not None) + if waits: + concurrent.futures.wait(waits, timeout=backpressure_poll_s, + return_when=concurrent.futures.FIRST_COMPLETED) + else: + sleep(backpressure_poll_s) + except BaseException: + abort.set() + # Executor submit can fail before a future is installed. + # Every prelease remains in records, so none can disappear + # between preparation, registration and independent generate. + for item in (preparing, registering, cleaning): + if item is not None: + try: + result = item[0].result() + if item is preparing: + for record, prepared in zip(item[1], result): + record["prepared"] = prepared + except BaseException: + pass + cleanup_errors = [] + for future, record in futures.items(): + try: + refs = future.result() + except BaseException: + pass + else: + for ref in refs: + try: + abort_feature_ref(feature_store, ref, + reason="producer-driver-failed-before-publication") + except BaseException as exc: + cleanup_errors.append(f"{type(exc).__name__}: {exc}") + record["settled"] = not future.cancelled() + for record in records: + cleaned = False + try: + if "prepared" in record: + clean_batch([record]) + cleaned = True + except BaseException as exc: + cleanup_errors.append(f"{type(exc).__name__}: {exc}") + finally: + if not record["settled"]: + w.fail_leased(record["tasks"], "producer-driver-failed") + if cleaned: + release_credit(record) + if cleanup_errors: + logger.error("producer exact-attempt cleanup remains pending: %s", cleanup_errors) + raise + last_ingest_log = {"t": None} def ingest_prompt_batch(epoch: int, batch_index: int, epoch_prompts) -> None: @@ -1488,6 +1794,11 @@ def run_worker_guarded(w) -> None: f"all rollout workers exited with {len(dead)} dropped " f"as dead and prompts remaining — dead workers: {dead}" ) + if inactive and not stopped and remaining: + raise RuntimeError( + "rollout workers stopped or draining with prompts remaining: " + f"{inactive}" + ) if st["prompts_failed"] and not stopped: raise RuntimeError( "producer finished with " @@ -1567,6 +1878,8 @@ def build_disagg_online_consumer( async_ack: Optional[bool] = None, consumer_dispatch: str = "cost_balanced", dispatch_num_anchors: int = 512, + ref_poll_s: float = 0.1, + distributor_poll_s: float = 0.05, ): """Consumer (trainer) side of an ONLINE disaggregated run. @@ -1744,6 +2057,7 @@ def build_disagg_online_consumer( skip_ids=skip_ids, requeued_ids=requeued_ids, idle_timeout_s=idle_timeout_s, + poll_s=distributor_poll_s, ) inbox_server_url = os.environ.get("DISAGG_INBOX_SERVER_URL") if inbox_server_url: @@ -1801,7 +2115,7 @@ def build_disagg_online_consumer( inbox = RemoteInboxChannel(inbox_server_url, dp_rank) else: inbox = InboxChannel(RefDistributor.inbox_path(inbox_dir, dp_rank)) - queue = StreamingRefQueue(inbox, idle_timeout_s=idle_timeout_s) + queue = StreamingRefQueue(inbox, idle_timeout_s=idle_timeout_s, poll_s=ref_poll_s) drain_state = {"attempted": False} fit_failure = {"error": None} diff --git a/specforge/launch_plan.py b/specforge/launch_plan.py index 436a84ae85..03569cea09 100644 --- a/specforge/launch_plan.py +++ b/specforge/launch_plan.py @@ -772,6 +772,22 @@ def _managed_local_services( argv.extend(cfg.model.sglang_extra_args) argv.extend(server.extra_args) gpu_put = server.gpu_put if server.gpu_put is not None else default_gpu_put + hs_env = {} + if deployment.hs_capture: + if gpu_put: + raise ValueError("HS capture supports host publication only; disable gpu_put") + if mooncake.arena_bytes is not None: + raise ValueError("HS capture does not support NVLink arena publication") + gpu_put = False + hs_env = { + "SPECFORGE_HS_CAPTURE": "1", + "SPECFORGE_HS_CACHE": "1" if deployment.hs_prefix_cache else "0", + **{ + "SPECFORGE_HS_" + name.upper(): str(getattr(deployment, "hs_" + name)) + for name in ("chunk_tokens", "cache_bytes", "pinned_bytes", + "output_bytes", "device_bytes", "max_workers", "attempt_capacity") + }, + } python_env: dict[str, Optional[str]] = {} if server.python_executable: python_env = {"PYTHONPATH": None, "VIRTUAL_ENV": None} @@ -789,7 +805,9 @@ def _managed_local_services( else {} ), **({"SPECFORGE_SPEC_CAPTURE": "1"} if server_capture is not None else {}), - "FLASHINFER_DISABLE_VERSION_CHECK": "1", + **hs_env, + # Installation/runtime compatibility is checked by the real guard. + "FLASHINFER_DISABLE_VERSION_CHECK": None, "MOONCAKE_GLOBAL_SEGMENT_SIZE": str(mooncake.global_segment_size_bytes), "MOONCAKE_LOCAL_BUFFER_SIZE": str(mooncake.local_buffer_size_bytes), **( diff --git a/specforge/lr_scheduler.py b/specforge/lr_scheduler.py index c375842bd0..b0db9b5cc0 100644 --- a/specforge/lr_scheduler.py +++ b/specforge/lr_scheduler.py @@ -1,3 +1,4 @@ +import math from warnings import warn from torch.optim.lr_scheduler import CosineAnnealingLR as _CosineAnnealingLR @@ -119,6 +120,40 @@ def __init__( super().__init__(optimizer, warmup_steps, base_scheduler, last_epoch=last_epoch) +class TorchSpecCosineWarmupLR(_LRScheduler): + """Opt-in match for the archived TorchSpec cosine schedule. + + With warmup, step zero is zero LR and warmup reaches peak at warmup_steps. + Without warmup, step zero is the peak. Cosine reaches eta_min at total_steps. + The default SpecForge path is unchanged. + """ + + def __init__( + self, optimizer, total_steps, warmup_steps=0, eta_min=0.0, last_epoch=-1 + ): + if not 0 <= warmup_steps < total_steps: + raise ValueError("warmup_steps must be in [0, total_steps)") + if eta_min < 0 or any(eta_min > g["lr"] for g in optimizer.param_groups): + raise ValueError("eta_min must be between zero and each peak LR") + self.total_steps = int(total_steps) + self.warmup_steps = int(warmup_steps) + self.eta_min = float(eta_min) + super().__init__(optimizer, last_epoch) + + def get_lr(self): + step = self.last_epoch + if self.warmup_steps and step <= self.warmup_steps: + return [peak * step / self.warmup_steps for peak in self.base_lrs] + progress = min( + 1.0, + max( + 0.0, (step - self.warmup_steps) / (self.total_steps - self.warmup_steps) + ), + ) + coeff = 0.5 * (math.cos(math.pi * progress) + 1.0) + return [self.eta_min + coeff * (peak - self.eta_min) for peak in self.base_lrs] + + class _FlatLR(_LRScheduler): """Keep every parameter group at its configured base learning rate.""" diff --git a/specforge/optimizer.py b/specforge/optimizer.py index 6fd4b6f467..4ebbabb52f 100644 --- a/specforge/optimizer.py +++ b/specforge/optimizer.py @@ -4,7 +4,11 @@ import torch import torch.distributed as dist -from specforge.lr_scheduler import ConstantWarmupLR, CosineAnnealingWarmupLR +from specforge.lr_scheduler import ( + ConstantWarmupLR, + CosineAnnealingWarmupLR, + TorchSpecCosineWarmupLR, +) from specforge.utils import print_on_rank0 logger = logging.getLogger(__name__) @@ -49,6 +53,7 @@ def __init__( warmup_ratio=0.015, lr_scheduler="cosine", offload_master=False, + min_lr=0.0, ): # defaults copied from EAGLE traineagle3 ds_config.json self.model = model @@ -84,6 +89,7 @@ def __init__( scheduler_types = { "constant": ConstantWarmupLR, "cosine": CosineAnnealingWarmupLR, + "torchspec_cosine": TorchSpecCosineWarmupLR, } if lr_scheduler not in scheduler_types: raise ValueError( @@ -91,10 +97,12 @@ def __init__( f"expected one of {sorted(scheduler_types)}" ) self.lr_scheduler_type = lr_scheduler + scheduler_kwargs = {} if lr_scheduler == "constant" else {"eta_min": min_lr} self.scheduler = scheduler_types[lr_scheduler]( self.optimizer, total_steps=total_steps, warmup_steps=int(warmup_ratio * total_steps), + **scheduler_kwargs, ) def configure_grad_norm_reduction( diff --git a/specforge/runtime/control_plane/controller.py b/specforge/runtime/control_plane/controller.py index 53a577f50c..c5cdb6dc57 100644 --- a/specforge/runtime/control_plane/controller.py +++ b/specforge/runtime/control_plane/controller.py @@ -22,6 +22,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import dataclasses import threading import uuid @@ -250,10 +252,12 @@ def reconcile_on_restart( # never have materialized refs that belonged to another DP # rank, so seed generation/key bookkeeping from the durable # committed ref before asking it to remove remote objects. + if ref is None: + raise RuntimeError("cannot reconcile released sample without its committed ref") adopt = getattr(feature_store, "adopt", None) - if callable(adopt) and ref is not None: + if callable(adopt): adopt(ref) - feature_store.abort(sample_id, reason="reconciled-released") + abort_feature_ref(feature_store, ref, reason="reconciled-released") continue if ref is not None: self.sample_queue.put([ref]) diff --git a/specforge/runtime/data_plane/feature_dataloader.py b/specforge/runtime/data_plane/feature_dataloader.py index a01186e960..a57e02b9bd 100644 --- a/specforge/runtime/data_plane/feature_dataloader.py +++ b/specforge/runtime/data_plane/feature_dataloader.py @@ -17,6 +17,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import os import queue as queue_module import threading @@ -350,7 +352,7 @@ def _settle_incomplete_queue_batch(self, refs: List[SampleRef]) -> None: adopt = getattr(self.store, "adopt", None) if callable(adopt): adopt(ref) - self.store.abort(ref.sample_id, reason=reason) + abort_feature_ref(self.store, ref, reason=reason) except Exception as exc: # settle the full tail, then fail loudly cleanup_errors.append(f"{ref.sample_id}: {exc}") if cleanup_errors: diff --git a/specforge/runtime/data_plane/feature_store.py b/specforge/runtime/data_plane/feature_store.py index 372cc969db..588608e548 100644 --- a/specforge/runtime/data_plane/feature_store.py +++ b/specforge/runtime/data_plane/feature_store.py @@ -166,6 +166,15 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: return {"force_freed": 0, "force_freed_bytes": 0, "release_pending": 0} +def abort_feature_ref(store, ref: SampleRef, *, reason: str) -> None: + """Abort the exact ref held by a delayed callback, with legacy fallback.""" + abort_ref = getattr(store, "abort_ref", None) + if callable(abort_ref): + abort_ref(ref, reason=reason) + else: + store.abort(ref.sample_id, reason=reason) + + def drain_feature_store_removals( store: FeatureStore, *, diff --git a/specforge/runtime/data_plane/mooncake_store.py b/specforge/runtime/data_plane/mooncake_store.py index ee1d478ac1..6ed4cac73a 100644 --- a/specforge/runtime/data_plane/mooncake_store.py +++ b/specforge/runtime/data_plane/mooncake_store.py @@ -58,6 +58,9 @@ from __future__ import annotations +from contextlib import contextmanager, ExitStack +from dataclasses import dataclass + import logging import os import threading @@ -538,6 +541,18 @@ def health(self) -> Dict[str, Any]: } +@dataclass(frozen=True) +class _CaptureBeginIntent: + """Stable retry identity and its batch-scoped singleflight owner.""" + protocol: Any + capability: dict + payload: dict + tokens: tuple + attempt_id: str + names: tuple + flight: Any + + class MooncakeFeatureStore(FeatureStore): """A disaggregated :class:`FeatureStore` backed by the Mooncake store. @@ -671,6 +686,17 @@ def __init__( # can reclaim those provisional objects; a successful adopt clears it. self._external_provisional: Dict[Tuple[str, int], List[str]] = {} self._active_leases: Dict[str, FeatureHandle] = {} + self._capture_fences = {} + self._capture_quarantined_registrations = set() + self._capture_handle_tokens = {} + self._capture_ack_pending = {} + self._capture_pending_begin = {} + self._terminal_cleanup_authority = None + self._terminal_cleanup_lock = threading.RLock() + self._capture_cleanup_local = threading.local() + self._capture_read_local = threading.local() + self._capture_read_intents = {} + self._capture_reads = {} # Samples whose remote remove() failed. gc() performs bounded # steady-state retries; lifecycle drain either removes them or raises. self._release_pending: Dict[str, int] = {} @@ -682,16 +708,356 @@ def __init__( # frees within a run (empty in retain_on_release/offline mode); a durable # shared index would own this in the online multi-node follow-up. self._freed: set = set() + self._freed_capture_attempts: set = set() self._lock = threading.RLock() self._counter = 0 self._gen_counter = 0 self._stats = {"force_freed": 0, "force_freed_bytes": 0} + def _begin_capture_read(self, ref): + fence = self._capture_from_ref(ref) + if fence is None: + return None + owner = uuid.uuid4().hex + intent = {"fence": fence, "owner": owner, "token": None, "unsafe_fetch": False, + "handle": None, "release_requested": False, "active": True} + with self._lock: + if self._capture_fences.get(ref.sample_id) is not fence or self._capture_ref_freed_locked(ref.sample_id, fence.wire): + raise KeyError('capture read ownership was released') + self._capture_read_intents[owner] = intent + # Keep intent before HTTP: lost acquire replies recover the same owner. + try: + with fence.lock: + intent["token"] = owner if fence.transferred else fence.acquire_read(owner) + except BaseException as exc: + from specforge.inference.capture_protocol import CaptureControlRejected + intent["active"] = False + if isinstance(exc, CaptureControlRejected): + with self._lock: + self._capture_read_intents.pop(owner, None) + raise + return intent + + def _release_capture_read(self, intent): + if intent["unsafe_fetch"]: + raise RuntimeError("capture receive drain unknown; read ownership retained") + fence = intent["fence"] + with fence.lock: + if not fence.transferred: + if intent["token"] is None: + intent["token"] = fence.acquire_read(intent["owner"]) + fence.release_read(intent["owner"], intent["token"]) + with self._lock: + self._capture_read_intents.pop(intent["owner"], None) + handle = intent["handle"] + if handle is not None: + self._capture_reads.pop(handle.lease_token, None) + if intent["release_requested"]: + self._active_leases.pop(handle.lease_token, None) + self._capture_handle_tokens.pop(handle.lease_token, None) + sid = handle.sample_id + if (not self.retain_on_release and self._capture_fences.get(sid) is fence + and not self._still_leased_locked(sid, handle.generation) + and not any(other["fence"] is fence for other in self._capture_read_intents.values())): + self._mark_freed_locked(sid, handle.generation) + self._release_pending.setdefault(sid, 0) + + def _recover_capture_reads(self, *, require_drained=False): + from specforge.inference.capture_protocol import CaptureControlRejected + with self._lock: + intents = list(self._capture_read_intents.values()) + failures = [] + for intent in intents: + if intent.get("active"): + continue + if intent["unsafe_fetch"]: + failures.append(intent["owner"] + ": receive drain unknown") + continue + if intent["handle"] is None or intent["release_requested"]: + try: + self._release_capture_read(intent) + except CaptureControlRejected as exc: + if intent["token"] is None: + # No native read started; authoritative refusal proves + # this owner has no active permit. Retire this intent only. + with self._lock: + self._capture_read_intents.pop(intent["owner"], None) + else: + intent["recovery_error"] = str(exc) + failures.append(intent["owner"] + ": " + str(exc)) + except Exception as exc: + intent["recovery_error"] = str(exc) + failures.append(intent["owner"] + ": " + str(exc)) + if require_drained and failures: + raise RuntimeError("capture read ownership not drained; permits retained: " + repr(failures)) + + def begin_capture_attempts(self, protocol, capability, requests): + """Reserve one batch while unrelated identities contact servers in parallel. + + A flight belongs to these immutable intents, not to the shared store. + Recovery can wait for this exact batch without serializing other servers. + """ + flight = threading.RLock() + # Prepare long token snapshots outside the metadata critical section. + intents = [(str(payload["sample_id"]), _CaptureBeginIntent( + protocol, dict(capability), dict(payload), tuple(tokens), attempt_id, + tuple(names), flight)) for payload, tokens, attempt_id, names in requests] + ids = [sid for sid, _ in intents] + with flight: + with self._lock: + if len(set(ids)) != len(ids) or any(sid in self._capture_pending_begin for sid in ids): + raise RuntimeError("duplicate or pending capture begin intent") + if any(sid in self._capture_fences or sid in self._capture_ack_pending or sid in self._release_pending for sid in ids): + raise RuntimeError("prior capture attempt has not relinquished ownership") + self._capture_pending_begin.update(intents) + if not capability.get('begin_batch'): + return [self._recover_capture_begin(sid) for sid in ids] + from specforge.inference.capture_protocol import CaptureCapacityDeclined + try: + replies = protocol.begin_batch(capability, [(intent.payload, intent.tokens, intent.attempt_id) for _, intent in intents]) + except CaptureCapacityDeclined: + # The typed atomic refusal proves none of this batch acquired a + # server writer. Unknown/lost replies must retain their intents. + with self._lock: + for sid, intent in intents: + if self._capture_pending_begin.get(sid) is intent: + self._capture_pending_begin.pop(sid) + raise + for (sid, intent), wire in zip(intents, replies): + self._finish_capture_begin(sid, intent, wire) + return replies + + def begin_capture_attempt(self, protocol, capability, payload, tokens, attempt_id, feature_names): + return self.begin_capture_attempts(protocol, capability, [(payload, tokens, attempt_id, feature_names)])[0] + + def _finish_capture_begin(self, sid, intent, wire): + # Caller holds intent.flight. No waiter may retarget this operation to + # a replacement endpoint/epoch merely because it has the same sample id. + with self._lock: + if self._capture_pending_begin.get(sid) is not intent: + raise RuntimeError("capture begin intent changed before completion") + self.track_capture_fence(wire, intent.names, control_fn=intent.protocol.post) + self.track_external_attempt(sid, generation=1, feature_names=list(intent.names), + server_url=intent.protocol.url.removesuffix("/spec_capture/control")) + with self._lock: + if self._capture_pending_begin.get(sid) is intent: + self._capture_pending_begin.pop(sid, None) + + def _recover_capture_begin(self, sid, *, expected_intent=None): + with self._lock: + intent = self._capture_pending_begin.get(sid) + if expected_intent is not None and intent is not expected_intent: + return None + if intent is None: + return None # Fresh samples never wait for another endpoint's HTTP. + with intent.flight: + with self._lock: + if self._capture_pending_begin.get(sid) is not intent: + return None + wire = intent.protocol.begin(intent.capability, intent.payload, intent.tokens, intent.attempt_id) + self._finish_capture_begin(sid, intent, wire) + return wire + + @contextmanager + def _capture_cleanup_scope(self, sample_ids, *, expected_records=None): + # The initiating caller may already have crossed a network operation; + # carry its original identity rather than snapshotting a replacement. + with self._lock: + records = (dict(expected_records) if expected_records is not None else + {sid: self._capture_fences.get(sid) for sid in sorted(set(sample_ids))}) + local = self._capture_cleanup_local + previous = getattr(local, "records", {}) + depth = getattr(local, "depth", 0) + with ExitStack() as stack: + for sid, fence in records.items(): + if fence is None: + continue + stack.enter_context(fence.lock) + with self._lock: + still_current = self._capture_fences.get(sid) is fence + if still_current: + if fence.transferred: + with self._lock: + if any(intent['fence'] is fence for intent in self._capture_read_intents.values()): + raise RuntimeError("capture local readers or receive transfers have not drained") + try: + fence.acquire_cleanup() + except BaseException: + with self._lock: + if self._capture_fences.get(sid) is fence: + self._generation.setdefault(sid, int(fence.wire["gen"])) + self._sample_names.setdefault(sid, list(fence.names)) + self._sample_bytes.setdefault(sid, 0) + self._release_pending.setdefault(sid, 0) + raise + local.records = {**previous, **records} + local.depth = depth + 1 + try: + yield records + finally: + local.records, local.depth = previous, depth + if depth == 0: + self._flush_capture_acks(sample_ids=records) + + def _capture_cleanup_allows(self, sid): + records = getattr(self._capture_cleanup_local, "records", {}) + if sid not in records: + return self._capture_fences.get(sid) is None + return self._capture_fences.get(sid) is records[sid] + + def track_capture_fence(self, wire, feature_names, *, control_fn=None): + from specforge.inference.capture_protocol import WriterFence, http_control + sid = str(wire["sample_id"]) + if str(wire["store_id"]) != str(self.store_id) or int(wire["gen"]) != 1: + raise ValueError("capture fence store/generation mismatch") + with self._lock: + old = self._capture_fences.get(sid) + if old is not None: + if old.token != wire["reservation_token"]: + raise RuntimeError("capture sample still has an unclean attempt") + return old + if any(h.sample_id == sid for h in self._active_leases.values()): + raise RuntimeError("capture sample has active consumer leases") + fence = WriterFence(dict(wire), tuple(feature_names), post=control_fn or http_control) + if wire.get('publication_ownership') == 'attempt-v1': + fence.accept_transfer(wire) + self._capture_fences[sid] = fence + # Only a server-authorized new reservation can retire the old g1 tombstone. + self._freed.discard((sid, 1)) + return fence + + def _capture_from_ref(self, ref, *, validate_remote=False): + wire = ref.metadata.get("capture_fence") + if wire is None: + return + sid = ref.sample_id + if (wire.get("sample_id") != sid or str(wire.get("store_id")) != str(self.store_id) + or ref.metadata.get("generation") is None + or int(wire.get("gen", -1)) != int(ref.metadata["generation"])): + raise ValueError("capture ref sample/store/generation identity mismatch") + with self._lock: + old = self._capture_fences.get(sid) + if old is not None and old.token != wire["reservation_token"]: + raise KeyError("stale capture attempt ref") + if old is None and self._capture_ref_freed_locked(sid, wire): + raise KeyError("released capture attempt ref") + fence = self.track_capture_fence(wire, list(ref.feature_keys)) + if wire.get('publication_ownership') == 'attempt-v1': + fence.accept_transfer(wire) + if ref.feature_keys != wire['physical_keys']: + raise RuntimeError('capture ref physical keys mismatch') + if validate_remote and not fence.transferred: + status = fence.control("resolve", cancel=False) + if status.get("state") != "PUBLISHED" or not status.get("writer_drained"): + from specforge.inference.capture_protocol import CaptureControlRejected + raise CaptureControlRejected("cannot adopt an attempt that is not currently PUBLISHED") + # A returned complete SampleRef is emitted only after writer completion. + fence.writer_drained = True + fence.adopted = True + return fence + + def _resolve_capture_writer(self, sid, expected_token=None): + with self._lock: + fence = self._capture_fences.get(sid) + if fence is None: + return + if expected_token is not None and fence.token != expected_token: + return + with fence.lock: + fence.drain(cancel=True) + + def _flush_capture_acks(self, sample_ids=None): + if getattr(self._capture_cleanup_local, "depth", 0): + return + with self._lock: + pending = sorted((sid, fence) for sid, fence in self._capture_ack_pending.items() + if sample_ids is None or (sid in sample_ids and (not isinstance(sample_ids, dict) or sample_ids[sid] is fence))) + for sid, fence in pending: + with fence.lock: + fence.acknowledge() + with self._lock: + if self._capture_ack_pending.get(sid) is fence: + self._capture_ack_pending.pop(sid, None) + if self._capture_fences.get(sid) is fence: + self._capture_fences.pop(sid, None) + + def discard_capture_attempt(self, sample_id, *, server_epoch, attempt_id, + control_url, reservation_token=None): + """Retire only the original prepared writer, even after a peer retries. + + An unknown begin reply can be recovered only through its matching + immutable intent. The captured fence then stays the cleanup authority + through resolve, physical removal and acknowledgement. + """ + sid = str(sample_id) + with self._lock: + intent = self._capture_pending_begin.get(sid) + if intent is not None and not ( + intent.protocol.url == control_url + and intent.capability["server_epoch"] == server_epoch + and intent.attempt_id == attempt_id + ): + intent = None + if intent is not None: + self._recover_capture_begin(sid, expected_intent=intent) + with self._lock: + fence = self._capture_fences.get(sid) + if fence is None or any(fence.wire.get(name) != value for name, value in ( + ("server_epoch", server_epoch), ("attempt_id", attempt_id), + ("control_url", control_url), + )) or (reservation_token is not None and fence.token != reservation_token): + return + if fence.adopted and sid in self._generation and sid not in self._release_pending: + raise RuntimeError("cannot discard an adopted live capture attempt") + with self._capture_cleanup_scope([sid], expected_records={sid: fence}): + with self._lock: + if self._capture_fences.get(sid) is not fence: + return + if fence.adopted and sid in self._generation and sid not in self._release_pending: + raise RuntimeError("cannot discard an adopted live capture attempt") + self._abort_impl(sid, reason="capture-prepared-discard", attempt_token=fence.token) + self.drain_sample_removals([sid]) + with self._lock: + if (self._capture_fences.get(sid) is fence + or self._capture_ack_pending.get(sid) is fence): + raise RuntimeError("prepared capture cleanup has not drained") + + def prepare_capture_retry(self, sample_id): + sid = str(sample_id) + self._recover_capture_begin(sid) + with self._lock: + fence = self._capture_fences.get(sid) + if any(h.sample_id == sid for h in self._active_leases.values()): + raise RuntimeError("cannot replace capture with active consumer lease") + adopted_live = fence is not None and fence.adopted and sid in self._generation and sid not in self._release_pending + if adopted_live and fence.transferred: + raise RuntimeError("cannot replace an adopted live capture sample before ownership handoff") + if adopted_live: + with fence.lock: + state = fence.control("resolve", cancel=False) + if state.get("state") != "RETIRED": + raise RuntimeError("cannot replace an adopted live capture sample") + fence.remote_retired = fence.writer_drained = fence.removed = fence.acknowledged = True + if fence is not None: + self._resolve_capture_writer(sid, fence.token) + self.abort(sid, reason="capture-retry", attempt_token=fence.token) + self.drain_sample_removals([sid]) + self._flush_capture_acks() + with self._lock: + if sid in self._capture_fences or sid in self._capture_ack_pending or sid in self._release_pending: + raise RuntimeError("prior capture cleanup has not drained") + # -- keys -------------------------------------------------------------- def _tkey(self, sample_id: str, gen: int, name: str) -> str: # generation lives in the key (like SharedDirFeatureStore's filename gen): # a re-put writes a new-gen key set and removes the old, so a stale ref's # keys are gone -> get() raises (B5), no payload-carried gen needed. + fence = self._capture_fences.get(sample_id) + if fence is not None and fence.wire.get('object_key_layout') == 'attempt-v1': + from specforge.inference.capture_protocol import attempt_keys + if name not in fence.names: + raise KeyError("feature not owned by capture attempt") + return attempt_keys(fence.wire, (name,))[name] return f"{self.store_id}/{sample_id}/g{gen}/{name}" # -- store wrappers (status-code aware) -------------------------------- @@ -753,16 +1119,29 @@ def close(self) -> None: """ if self._closed: return + with self._lock: + if any(intent["active"] or (intent["handle"] is not None and not intent["release_requested"]) + for intent in self._capture_read_intents.values()): + raise RuntimeError("cannot close capture transport with live readers") if self._receive_pool is not None: self._receive_pool.close() if self._owns_store: - rc = self._store.close() - if rc is not None and int(rc) != 0: - raise RuntimeError(f"Mooncake transport close failed ({rc})") + if not getattr(self, "_transport_stopped", False): + rc = self._store.close() + if rc is not None and int(rc) != 0: + raise RuntimeError(f"Mooncake transport close failed ({rc})") + self._transport_stopped = True if self._receive_pool is not None: self._receive_pool.close(transport_stopped=True) self._quarantined_buffers.clear() self._quarantined_bytes = 0 + self._capture_quarantined_registrations.clear() + # Successful transport shutdown proves every quarantined read has + # physically drained. An injected/shared transport has no such proof. + with self._lock: + for intent in self._capture_read_intents.values(): + intent["unsafe_fetch"] = False + self._recover_capture_reads(require_drained=True) self._closed = True def _fetch_tensor(self, key: str, spec, device="cpu") -> torch.Tensor: @@ -774,6 +1153,10 @@ def _fetch_tensor(self, key: str, spec, device="cpu") -> torch.Tensor: try: return self._fetch_tensor_once(key, spec, device) except _MooncakeGetError as exc: + if getattr(self._capture_read_local, "intent", None) is not None: + # A successful retry cannot prove a prior transfer drained. + # Keep the exact read permission until transport shutdown. + raise if exc.status not in _RETRYABLE_GET_STATUSES or attempt + 1 == attempts: raise delay = _GET_RETRY_DELAYS_S[attempt] @@ -793,10 +1176,11 @@ def _fetch_tensor_once(self, key: str, spec, device) -> torch.Tensor: dst = _alloc_from_spec(spec) try: self._store_get_tensor(key, dst) - except _MooncakeGetError: - # Quarantine the final failed attempt too: propagating its - # exception must not free storage that a late transfer can use. - self._quarantine_buffer(dst) + except BaseException as exc: + # Native get_into can raise before confirming physical drain. + # A failed fenced read keeps both storage and registration. + if isinstance(exc, _MooncakeGetError) or getattr(self._capture_read_local, "intent", None) is not None: + self._quarantine_buffer(dst) raise return dst dtype = _TORCH_DTYPES.get(spec.dtype) @@ -861,14 +1245,21 @@ def _store_get_tensor(self, key: str, out: torch.Tensor) -> None: self._store.register_buffer(out.data_ptr(), nb) except Exception: # pragma: no cover - some builds auto-register pass + drained = False try: rc = self._store.get_into(key, out.data_ptr(), nb) + _check_get_result(key, rc, nb) + drained = True finally: - try: - self._store.unregister_buffer(out.data_ptr()) - except Exception: # pragma: no cover - pass - _check_get_result(key, rc, nb) + if drained or getattr(self._capture_read_local, "intent", None) is None: + try: + self._store.unregister_buffer(out.data_ptr()) + except Exception: # pragma: no cover + pass + else: + # Unregistering before an uncertain DMA completes is unsafe. + # The owned transport's successful close is the release proof. + self._capture_quarantined_registrations.add(out.data_ptr()) def _store_remove(self, key: str, *, force: bool = False) -> bool: """Best-effort physical free. Returns True once the key is gone. @@ -985,6 +1376,7 @@ def adopt(self, sample_ref: SampleRef) -> None: raise ValueError( f"cannot adopt {sample_ref.sample_id}: ref carries no generation" ) + self._capture_from_ref(sample_ref, validate_remote=True) self._locate(sample_ref) with self._lock: gen = int(gen) @@ -1024,8 +1416,31 @@ def track_external_attempt( ) self._external_provisional[(str(sample_id), int(generation))] = names - def discard_external_attempts( - self, *, reason: str = "unadopted-external-capture" + def discard_external_attempts(self, *, reason="unadopted-external-capture"): + self._recover_capture_reads(require_drained=True) + with self._lock: + pending_begins = list(self._capture_pending_begin) + for sid in pending_begins: + self._recover_capture_begin(sid) + with self._lock: + snapshot = [(identity, names, self._capture_fences.get(identity[0])) + for identity, names in self._external_provisional.items()] + discarded = 0 + legacy = [] + for (sid, gen), names, expected in snapshot: + if expected is None: + legacy.append(((sid, gen), names)) + continue + with expected.lock: + with self._lock: + if self._capture_fences.get(sid) is not expected or (sid, gen) not in self._external_provisional: + continue + self.abort(sid, reason=reason, attempt_token=expected.token) + discarded += 1 + return discarded + (self._discard_external_attempts_legacy(reason=reason, expected_attempts=legacy) if legacy else 0) + + def _discard_external_attempts_legacy( + self, *, reason: str = "unadopted-external-capture", expected_attempts=() ) -> int: """Abort server writes that never produced an adoptable response. @@ -1035,18 +1450,21 @@ def discard_external_attempts( shutdown path. Physical-remove failures remain visible to :meth:`drain_pending_removals`. """ - with self._lock: - attempts = list(self._external_provisional.items()) + attempts = expected_attempts errors: Dict[str, str] = {} discarded = 0 for (sample_id, generation), names in attempts: + with self._lock: + fence = self._capture_fences.get(sample_id) + if fence is not None: + continue with self._lock: attempt_key = (sample_id, generation) # A response can be adopted after the snapshot but before this # attempt is visited. adopt() removes the provisional entry # under the same lock, so never delete that now-live sample. - if attempt_key not in self._external_provisional: + if self._external_provisional.get(attempt_key) is not names or self._capture_fences.get(sample_id) is not None: continue current = self._generation.get(sample_id) if current is not None: @@ -1077,6 +1495,7 @@ def discard_external_attempts( continue self._external_provisional.pop(attempt_key, None) discarded += 1 + self._flush_capture_acks() if errors: raise RuntimeError( "could not discard all provisional external captures: " f"{errors}" @@ -1084,7 +1503,31 @@ def discard_external_attempts( return discarded # -- read -------------------------------------------------------------- - def get( + def get(self, sample_ref: SampleRef, *, device="cpu", names=None): + self.auth.check(self._credential) + self._recover_capture_reads() + intent = self._begin_capture_read(sample_ref) + previous = getattr(self._capture_read_local, "intent", None) + self._capture_read_local.intent = intent + try: + result = self._get_impl(sample_ref, device=device, names=names) + except BaseException: + if intent is not None: + intent["active"] = False + if not intent["unsafe_fetch"]: + intent["release_requested"] = True + self._release_capture_read(intent) + raise + finally: + self._capture_read_local.intent = previous + if intent is not None: + with self._lock: + intent["handle"] = result[1] + self._capture_reads[result[1].lease_token] = intent + intent["active"] = False + return result + + def _get_impl( self, sample_ref: SampleRef, *, @@ -1092,10 +1535,11 @@ def get( names: Optional[List[str]] = None, ) -> Tuple[Dict[str, torch.Tensor], FeatureHandle]: self.auth.check(self._credential) + self._capture_from_ref(sample_ref) sid = sample_ref.sample_id ref_gen = sample_ref.metadata.get("generation") with self._lock: - if ref_gen is not None and (sid, int(ref_gen)) in self._freed: + if ref_gen is not None and self._capture_ref_freed_locked(sid, sample_ref.metadata.get("capture_fence") or {"gen": ref_gen}): # logically freed here; the remote bytes may still linger under # Mooncake's read-lease, but this ref must not resolve (B5) raise KeyError( @@ -1126,6 +1570,9 @@ def get( lease_token=f"{sid}:{self._counter}", ) self._active_leases[handle.lease_token] = handle + fence = self._capture_fences.get(sid) + if fence is not None: + self._capture_handle_tokens[handle.lease_token] = fence.token return out, handle def _locate(self, sample_ref: SampleRef) -> None: @@ -1168,7 +1615,13 @@ def _get_tensors( f"(freed, stale, or never written)" ) # fresh alloc (or a pooled slot copied out) -> clone-on-fetch for free (B5) - out[n] = self._fetch_tensor(key, spec, device) + try: + out[n] = self._fetch_tensor(key, spec, device) + except BaseException: + intent = getattr(self._capture_read_local, "intent", None) + if intent is not None: + intent["unsafe_fetch"] = True + raise return out, gen # -- lifetime ---------------------------------------------------------- @@ -1188,6 +1641,19 @@ def _try_physical_free( disable that probe because probing a still-live key would renew its lease and make every following remove fail again. """ + fence = self._capture_fences.get(sample_id) + expected = getattr(self._capture_cleanup_local, "records", {}).get(sample_id) + if expected is not None and fence is not expected: + return False + if fence is not None: + if expected is not fence: + return False + if fence.remote_retired: + return True # Other store retired old attempt: discard LOCAL metadata only. + if not fence.writer_drained or (not fence.transferred and not fence.cleanup_token): + return False + if fence is not None and fence.transferred and any(intent['fence'] is fence for intent in self._capture_read_intents.values()): + return False gen = self._generation.get(sample_id) if gen is None: return True # nothing tracked to remove (already freed) @@ -1203,6 +1669,14 @@ def _try_physical_free( def _free_bookkeeping_locked(self, sample_id: str) -> int: """Drop in-process tracking for a sample. Returns bytes accounted freed.""" + fence = self._capture_fences.get(sample_id) + if fence is not None: + if not self._capture_cleanup_allows(sample_id): + raise RuntimeError("stale cleanup cannot mutate replacement bookkeeping") + if not fence.writer_drained: + raise RuntimeError("free bookkeeping before capture writer drain") + fence.removed = True + self._capture_ack_pending[sample_id] = fence generation = self._generation.get(sample_id) nbytes = self._sample_bytes.pop(sample_id, 0) self._generation.pop(sample_id, None) @@ -1223,6 +1697,31 @@ def _still_leased_locked(self, sample_id: str, generation: Optional[int]) -> boo def release(self, handle: FeatureHandle, *, reason: str = "consumed") -> None: with self._lock: + intent = self._capture_reads.get(handle.lease_token) + expected = intent["fence"] if intent is not None else self._capture_fences.get(handle.sample_id) + if intent is None and handle.lease_token not in self._active_leases and expected is not None: + return + if intent is not None: + intent["release_requested"] = True + self._release_capture_read(intent) + if self.retain_on_release: + return + with self._lock: + if expected.transferred and any(other["fence"] is expected for other in self._capture_read_intents.values()): + return # The final local reader owns the eventual deletion. + with self._capture_cleanup_scope([handle.sample_id], expected_records={handle.sample_id: expected}): + return self._release_impl(handle, reason=reason) + + def _release_impl(self, handle: FeatureHandle, *, reason: str = "consumed") -> None: + with self._lock: + if not self._capture_cleanup_allows(handle.sample_id): + return + expected = self._capture_handle_tokens.pop(handle.lease_token, None) + fence = self._capture_fences.get(handle.sample_id) + if expected is not None and (fence is None or fence.token != expected): + self._active_leases.pop(handle.lease_token, None) + return + self._active_leases.pop(handle.lease_token, None) if self.retain_on_release: return # offline re-iterable set: keep for the next epoch @@ -1232,23 +1731,70 @@ def release(self, handle: FeatureHandle, *, reason: str = "consumed") -> None: return # stale lease -> no-op if self._still_leased_locked(sid, cur): return - self._freed.add((sid, handle.generation)) # immediate logical free + self._mark_freed_locked(sid, handle.generation) # immediate logical free if self._try_physical_free(sid): self._free_bookkeeping_locked(sid) else: # remote free deferred (lease) / failed -> gc() retries self._release_pending.setdefault(sid, 0) - def abort(self, sample_id: str, *, reason: str = "aborted") -> None: + self._flush_capture_acks() + + def abort_ref(self, ref: SampleRef, *, reason="aborted") -> None: + wire = ref.metadata.get("capture_fence") + if wire is not None: + if wire.get("sample_id") != ref.sample_id or str(wire.get("store_id")) != str(self.store_id) or not wire.get("reservation_token"): + raise ValueError("capture cleanup ref identity mismatch") + self.abort(ref.sample_id, reason=reason, attempt_token=wire["reservation_token"]) + return + # A legacy callback carries no authority over a later negotiated + # attempt. Keep the generation comparison and delete under one lock. with self._lock: + if ref.sample_id in self._capture_fences: + return + generation = ref.metadata.get("generation") + if generation is None or self._generation.get(ref.sample_id) != int(generation): + return + self.abort(ref.sample_id, reason=reason) + + def abort(self, sample_id: str, *, reason: str = "aborted", attempt_token=None) -> None: + with self._lock: + current = self._capture_fences.get(sample_id) + if attempt_token is not None and (current is None or current.token != attempt_token): + return + with self._capture_cleanup_scope([sample_id], expected_records={sample_id: current}): + return self._abort_impl(sample_id, reason=reason, attempt_token=attempt_token) + + def _abort_impl(self, sample_id: str, *, reason: str = "aborted", attempt_token=None) -> None: + with self._lock: + if not self._capture_cleanup_allows(sample_id): + return + fence = self._capture_fences.get(sample_id) + if attempt_token is not None and (fence is None or fence.token != attempt_token): + return + if fence is not None: + self._resolve_capture_writer(sample_id, attempt_token) + with self._lock: + if fence is not None: + if self._capture_fences.get(sample_id) is not fence: + return + if any(h.sample_id == sample_id for h in self._active_leases.values()): + raise RuntimeError("cannot abort capture with active consumer lease") + # Lost responses have no adopted generation, but still own exact keys. + self._generation.setdefault(sample_id, int(fence.wire["gen"])) + self._sample_names.setdefault(sample_id, list(fence.names)) + self._sample_bytes.setdefault(sample_id, 0) + gen = self._generation.get(sample_id) if gen is not None: - self._freed.add((sample_id, gen)) # immediate logical free + self._mark_freed_locked(sample_id, gen) # immediate logical free if self._try_physical_free(sample_id): self._free_bookkeeping_locked(sample_id) else: self._release_pending.setdefault(sample_id, 0) + self._flush_capture_acks() + def drain_sample_removals( self, sample_ids: List[str], @@ -1262,14 +1808,23 @@ def drain_sample_removals( Other pending samples may belong to prefetched, not-yet-durable batches and must remain available for crash replay. """ - return self._drain_removals( + result = self._drain_removals( sample_ids=sample_ids, max_attempts=max_attempts, retry_interval_s=retry_interval_s, sleep=sleep, ) + self._flush_capture_acks() + return result + def retry_sample_removals(self, sample_ids: List[str]) -> Dict[str, Any]: + with self._lock: + targets = [sid for sid in sample_ids if sid in self._release_pending or sid in self._capture_ack_pending] + with self._capture_cleanup_scope(targets): + return self._retry_sample_removals_impl(sample_ids) + + def _retry_sample_removals_impl(self, sample_ids: List[str]) -> Dict[str, Any]: """Try selected removals once, without probing or sleeping. The optimizer path calls this only for samples made durable at an @@ -1286,6 +1841,8 @@ def retry_sample_removals(self, sample_ids: List[str]) -> Dict[str, Any]: if sample_id in target_ids ] for sample_id in pending: + if not self._capture_cleanup_allows(sample_id): + continue physically_removed = self._try_physical_free( sample_id, force=True, @@ -1307,6 +1864,7 @@ def retry_sample_removals(self, sample_ids: List[str]) -> Dict[str, Any]: for sample_id in self._release_pending if sample_id in target_ids ] + self._flush_capture_acks() return { "removed": removed, "removed_bytes": removed_bytes, @@ -1331,14 +1889,24 @@ def drain_pending_removals( ``sleep`` is injectable so protocol tests can advance a fake lease clock without wall-clock delays. """ - return self._drain_removals( + result = self._drain_removals( sample_ids=None, max_attempts=max_attempts, retry_interval_s=retry_interval_s, sleep=sleep, ) - def _drain_removals( + self._flush_capture_acks() + return result + + def _drain_removals(self, *, sample_ids, **kwargs): + self._recover_capture_reads(require_drained=True) + with self._lock: + targets = [sid for sid in set(self._release_pending) | set(self._capture_ack_pending) if sample_ids is None or sid in sample_ids] + with self._capture_cleanup_scope(targets): + return self._drain_removals_impl(sample_ids=sample_ids, **kwargs) + + def _drain_removals_impl( self, *, sample_ids: Optional[List[str]], @@ -1351,6 +1919,11 @@ def _drain_removals( if retry_interval_s < 0: raise ValueError("retry_interval_s must be >= 0") target_ids = None if sample_ids is None else set(sample_ids) + with self._lock: + fenced = [sid for sid in self._release_pending if target_ids is None or sid in target_ids] + for sid in fenced: + self._resolve_capture_writer(sid) + self._flush_capture_acks() removed = removed_bytes = 0 last_errors: Dict[str, str] = {} attempts_run = 0 @@ -1371,6 +1944,8 @@ def _drain_removals( } final_attempt = attempt + 1 == max_attempts for sample_id in pending: + if not self._capture_cleanup_allows(sample_id): + continue try: physically_removed = self._try_physical_free( sample_id, @@ -1430,7 +2005,127 @@ def _drain_removals( f"{preview}{detail}" ) + def _capture_ref_freed_locked(self, sid, wire): + if wire.get('object_key_layout') == 'attempt-v1': + return (sid, wire.get('reservation_token')) in self._freed_capture_attempts + return (sid, int(wire['gen'])) in self._freed + + def _mark_freed_locked(self, sid, generation): + self._freed.add((sid, generation)) + fence = self._capture_fences.get(sid) + if fence is not None and fence.wire.get('object_key_layout') == 'attempt-v1': + self._freed_capture_attempts.add((sid, fence.token)) + + def _published_capture_locked(self, sample_id): + fence = self._capture_fences.get(sample_id) + return fence is not None and fence.transferred + + def relinquish_published(self, ref): + """Disown a possibly visible ref after the streaming publisher handoff. + + This drops producer metadata only. The durable ref and its designated + consumer/optimizer authority now own deletion of the immutable objects. + """ + wire = ref.metadata.get('capture_fence') or {} + if wire.get('publication_ownership') != 'attempt-v1': + return + sid = ref.sample_id + with self._lock: + fence = self._capture_fences.get(sid) + if fence is None: + return + with fence.lock, self._lock: + if self._capture_fences.get(sid) is not fence or fence.token != wire.get('reservation_token'): + return + if not fence.transferred or any(h.sample_id == sid for h in self._active_leases.values()) or any(intent['fence'] is fence for intent in self._capture_read_intents.values()): + raise RuntimeError('cannot relinquish capture with local reader ownership') + self._mark_freed_locked(sid, int(wire['gen'])) + for mapping in (self._generation, self._sample_names, self._sample_bytes, self._put_time, + self._release_pending, self._capture_ack_pending, self._capture_fences): + mapping.pop(sid, None) + self._external_provisional.pop((sid, int(wire['gen'])), None) + + def _validate_terminal_capture_ref(self, ref, wire): + """Pure validation must precede even an already-cleaned fast return.""" + from specforge.inference.capture_protocol import WriterFence + + if (wire.get("protocol_version") != 1 or wire.get("sample_id") != ref.sample_id + or str(wire.get("store_id")) != str(self.store_id) + or int(wire.get("gen", -1)) != 1 + or ref.metadata.get("generation") is None + or int(ref.metadata["generation"]) != int(wire["gen"])): + raise ValueError("terminal capture ref sample/store/generation identity mismatch") + if ref.feature_store_uri != f"mooncake://{self.store_id}/{ref.sample_id}": + raise ValueError("terminal capture ref canonical URI identity mismatch") + proof = WriterFence(dict(wire), tuple(ref.feature_keys)) + if not proof.accept_transfer(wire) or ref.feature_keys != wire.get("physical_keys"): + raise ValueError("terminal capture ref certificate/physical keys mismatch") + + def cleanup_terminal_refs(self, refs, *, consumer_done=False, producer_closed=False): + """Reclaim exact published attempts after both sides have finished. + + Normal handoff tombstones stay sealed. A separate metadata authority + may adopt immutable refs only after successful consumer drain and the + producer's joined-worker EOF. Failure/timeout alone proves neither. + The authority borrows this native backend and must never close it. + """ + refs = tuple(refs) + errors = [] + cleaned = 0 + with self._terminal_cleanup_lock: + for ref in refs: + wire = ref.metadata.get("capture_fence") or {} + immutable = (wire.get("object_key_layout") == "attempt-v1" or + wire.get("publication_ownership") == "attempt-v1") + try: + if not immutable: + # Legacy cleanup retains its existing remote reader / + # writer permits, even during an abnormal shutdown. + self.abort_ref(ref, reason="online-attempt-finished") + cleaned += 1 + continue + if consumer_done is not True or producer_closed is not True: + raise RuntimeError("terminal ownership unresolved: successful consumer_done and closed producer are required") + self._validate_terminal_capture_ref(ref, wire) + authority = self._terminal_cleanup_authority + if authority is None: + authority = MooncakeFeatureStore(store_id=self.store_id, store=self._store, + auth=self.auth, credential=self._credential, + max_release_attempts=self.max_release_attempts) + self._terminal_cleanup_authority = authority + token = wire.get("reservation_token") + with authority._lock: + already_cleaned = ((ref.sample_id, token) in authority._freed_capture_attempts + and ref.sample_id not in authority._capture_fences) + if not already_cleaned: + authority.adopt(ref) # Strict certificate + exact key validation. + authority.abort_ref(ref, reason="online-attempt-finished") + authority.drain_sample_removals([ref.sample_id]) + cleaned += 1 + except Exception as exc: + errors.append(f"{ref.sample_id}/{wire.get('reservation_token')}: {type(exc).__name__}: {exc}") + authority = self._terminal_cleanup_authority + if authority is not None and consumer_done is True and producer_closed is True: + try: + authority.drain_pending_removals() + except Exception as exc: + errors.append(f"terminal exact-key removal drain: {type(exc).__name__}: {exc}") + if errors: + raise RuntimeError("terminal published-ref cleanup unresolved: " + repr(errors)) + return {"refs_cleaned": cleaned} + def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: + self._recover_capture_reads() + current = self._clock() if now is None else now + with self._lock: + targets = set(self._release_pending) | set(self._capture_ack_pending) + if self.max_hold_age_s is not None: + targets.update(sid for sid, start in self._put_time.items() + if current-start > self.max_hold_age_s and not self._published_capture_locked(sid) and not self._still_leased_locked(sid, self._generation.get(sid))) + with self._capture_cleanup_scope(targets): + return self._gc_impl(now=current) + + def _gc_impl(self, *, now: Optional[float] = None) -> Dict[str, int]: now = self._clock() if now is None else now freed = freed_bytes = 0 with self._lock: @@ -1440,9 +2135,12 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: sid for sid, t in list(self._put_time.items()) if now - t > self.max_hold_age_s + and not self._published_capture_locked(sid) and not self._still_leased_locked(sid, self._generation.get(sid)) ] for sid in stale: + if not self._capture_cleanup_allows(sid): + continue if self._try_physical_free(sid, confirm_absent_on_failure=False): freed_bytes += self._free_bookkeeping_locked(sid) freed += 1 @@ -1451,6 +2149,8 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: # Reconcile release-pending without an exists probe: is_exist grants # a read lease that would make the next remove fail (-706). for sid in list(self._release_pending): + if not self._capture_cleanup_allows(sid): + continue if self._release_pending[sid] >= self.max_release_attempts: # Keep the physical key metadata and surface the pending # sample. Lifecycle drain owns the final bounded retry and @@ -1465,6 +2165,7 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: self._release_pending[sid] = attempts self._stats["force_freed"] += freed self._stats["force_freed_bytes"] += freed_bytes + self._flush_capture_acks() return { "force_freed": freed, "force_freed_bytes": freed_bytes, @@ -1486,6 +2187,14 @@ def health(self) -> Dict[str, Any]: "backend": "mooncake", "resident_samples": len(self._generation), "provisional_external": len(self._external_provisional), + "capture_pending_begin": len(self._capture_pending_begin), + "capture_fences": len(self._capture_fences), + "capture_ack_pending": len(self._capture_ack_pending), + "capture_handle_attempts": len(self._capture_handle_tokens), + "capture_read_intents": len(self._capture_read_intents), + "capture_active_reads": len(self._capture_reads), + "capture_failed_reads": sum(bool(x["unsafe_fetch"]) for x in self._capture_read_intents.values()), + "capture_quarantined_registrations": len(self._capture_quarantined_registrations), "active_leases": len(self._active_leases), "resident_bytes": sum(self._sample_bytes.values()), "max_resident_bytes": self.max_resident_bytes, diff --git a/specforge/runtime/data_plane/ref_distributor.py b/specforge/runtime/data_plane/ref_distributor.py index 1ff78f5e04..005a0de7e3 100644 --- a/specforge/runtime/data_plane/ref_distributor.py +++ b/specforge/runtime/data_plane/ref_distributor.py @@ -44,6 +44,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import logging import os import threading @@ -473,7 +475,7 @@ def _finish(self) -> None: adopt = getattr(self.feature_store, "adopt", None) if callable(adopt): adopt(ref) - self.feature_store.abort(ref.sample_id, reason=reason) + abort_feature_ref(self.feature_store, ref, reason=reason) except Exception as exc: # best effort before the loud failure cleanup_errors.append(f"{ref.sample_id}: {exc}") self.source.mark_consumed( diff --git a/specforge/runtime/data_plane/streaming_ref_channel.py b/specforge/runtime/data_plane/streaming_ref_channel.py index 4f18b791d9..ff62de0702 100644 --- a/specforge/runtime/data_plane/streaming_ref_channel.py +++ b/specforge/runtime/data_plane/streaming_ref_channel.py @@ -225,6 +225,10 @@ def consumer_failure(self) -> Optional[str]: """Return the consumer failure reason, if one has been published.""" return self._read_sidecar(_CONSUMER_FAILED_SUFFIX) + def consumer_done(self) -> bool: + """Successful consumer drain proof; a failure sentinel always wins.""" + return os.path.exists(self.path + _CONSUMER_DONE_SUFFIX) and self.consumer_failure() is None + def consumer_stopped(self) -> bool: """Whether the remote consumer completed or failed.""" return os.path.exists(self.path + _CONSUMER_DONE_SUFFIX) or os.path.exists( diff --git a/specforge/training/assembly.py b/specforge/training/assembly.py index cd82c7b4eb..f37b83617f 100644 --- a/specforge/training/assembly.py +++ b/specforge/training/assembly.py @@ -265,6 +265,8 @@ def __call__(self, draft_module): return BF16Optimizer( draft_module, lr=t.learning_rate, + weight_decay=t.weight_decay, + min_lr=t.min_lr, max_grad_norm=t.max_grad_norm, warmup_ratio=t.warmup_ratio, lr_scheduler=t.lr_scheduler, @@ -356,6 +358,7 @@ def _prompt_cache_key( "path": source_path, "content_hash": content_hash, "max_length": cfg.data.max_length, + "overlength_policy": cfg.data.overlength_policy, "chat_template": cfg.data.chat_template, "tokenizer_chat_template_hash": _tokenizer_chat_template_hash(tokenizer), "is_preformatted": cfg.data.is_preformatted, @@ -410,6 +413,7 @@ def _prepare_prompts( tokenizer, chat_template=cfg.data.chat_template, max_length=cfg.data.max_length, + overlength_policy=cfg.data.overlength_policy, is_preformatted=cfg.data.is_preformatted, train_only_last_turn=cfg.data.train_only_last_turn, cache_dir=cfg.data.cache_dir, diff --git a/specforge/training/disaggregated.py b/specforge/training/disaggregated.py index 71f78b91ec..a2b2999c8e 100644 --- a/specforge/training/disaggregated.py +++ b/specforge/training/disaggregated.py @@ -604,6 +604,34 @@ def _producer_capture_metadata(cfg: Config, algorithm: AlgorithmRegistration): ) +def _cleanup_online_published_refs(store, channel): + """Run the existing terminal publication scan with exact-ref authority.""" + from specforge.runtime.data_plane.feature_store import abort_feature_ref + from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel + + reader = StreamingRefChannel(channel.path) + cleanup = getattr(store, "cleanup_terminal_refs", None) + errors = [] + while True: + refs = reader.poll(max_n=1024) + if not refs: + break + if callable(cleanup): + try: + cleanup(refs, consumer_done=channel.consumer_done(), + producer_closed=channel.is_closed()) + except Exception as exc: + errors.append(f"{type(exc).__name__}: {exc}") + else: + for ref in refs: + try: + abort_feature_ref(store, ref, reason="online-attempt-finished") + except Exception as exc: + errors.append(f"{ref.sample_id}: {type(exc).__name__}: {exc}") + if errors: + raise RuntimeError("terminal published-ref scan unresolved: " + repr(errors)) + + def _build_online( cfg: Config, *, @@ -703,6 +731,8 @@ def _build_online( request_input_adapter=input_adapter, target_model_version=cfg.model.target_model_path, server_capture=_server_capture(cfg), + hs_capture=cfg.deployment.disaggregated.hs_capture, + hs_prefix_cache=cfg.deployment.disaggregated.hs_prefix_cache, ) for url in _server_urls(cfg) ] @@ -741,6 +771,7 @@ def _build_online( feature_store_max_resident_bytes=( cfg.runtime.feature_store_max_resident_bytes ), + backpressure_poll_s=cfg.runtime.producer_backpressure_poll_s, peer_wait_timeout_s=peer_wait_timeout_s, ) @@ -775,18 +806,7 @@ def produce() -> int: ) cleanup_errors = [] try: - reader = StreamingRefChannel(channel_path) - while True: - refs = reader.poll(max_n=1024) - if not refs: - break - for ref in refs: - try: - store.abort(ref.sample_id, reason="online-attempt-finished") - except Exception as exc: - cleanup_errors.append( - f"{ref.sample_id}: {type(exc).__name__}: {exc}" - ) + _cleanup_online_published_refs(store, channel) except Exception as exc: cleanup_errors.append( f"published-ref scan: {type(exc).__name__}: {exc}" @@ -859,6 +879,8 @@ def produce() -> int: async_ack=_consumer_async_ack(cfg), consumer_dispatch=cfg.runtime.consumer_dispatch, dispatch_num_anchors=cfg.training.num_anchors, + ref_poll_s=cfg.runtime.consumer_ref_poll_s, + distributor_poll_s=cfg.runtime.ref_distributor_poll_s, ) return TrainingRun(trainer=trainer, on_finally=store.close) diff --git a/tests/plana/test_runtime_polling.py b/tests/plana/test_runtime_polling.py new file mode 100644 index 0000000000..fec2b9b276 --- /dev/null +++ b/tests/plana/test_runtime_polling.py @@ -0,0 +1,169 @@ +"""CPU gates for configurable polling without changing flow-control bounds.""" +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock + +from pydantic import ValidationError +import yaml + +from specforge.algorithms.builtin import builtin_algorithm_registry +from specforge.config import Config, load_config +from specforge.config.schema import RuntimeConfig +from specforge.launch import build_disagg_online_consumer +from specforge.runtime.control_plane.metadata_store import InMemoryMetadataStore +from specforge.runtime.data_plane.ref_distributor import RefDistributor +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from specforge.training.disaggregated import _build_online + + +ALGORITHM=builtin_algorithm_registry().resolve('dflash') +POLL_FIELDS=('producer_backpressure_poll_s','consumer_ref_poll_s','ref_distributor_poll_s') +DEFAULTS=(0.2,0.1,0.05) +TUNED=(0.011,0.013,0.017) +BOUNDS={'producer_lease':1,'producer_concurrency':1, + 'in_flight_high_watermark':4,'in_flight_low_watermark':2, + 'resident_high_watermark_bytes':512<<20,'resident_low_watermark_bytes':256<<20, + 'feature_store_max_resident_bytes':1<<30} + + +def payload(root,role,*,polling=None): + runtime=dict(BOUNDS) + if polling is not None:runtime.update(zip(POLL_FIELDS,polling)) + return {'model':{'target_model_path':'target','draft_model_config':'draft'}, + 'data':{'prompts_path':'prompts.jsonl','dataloader_num_workers':0}, + 'training':{'strategy':'dflash','role':role,'max_steps':1,'num_anchors':8, + 'seed':42,'prompt_seed':42},'runtime':runtime, + 'deployment':{'mode':'disaggregated','disaggregated':{ + 'control_dir':str(root),'backend':'mooncake','async_ack':False, + 'server_urls':['http://capture:30000']}}} + + +class PollingSchemaTests(unittest.TestCase): + def test_defaults_preserve_fixed_upstream_cadences_and_bounds(self): + runtime=RuntimeConfig() + self.assertEqual(tuple(getattr(runtime,name) for name in POLL_FIELDS),DEFAULTS) + self.assertEqual((runtime.in_flight_high_watermark,runtime.in_flight_low_watermark),(256,192)) + self.assertEqual(runtime.producer_concurrency,1) + + def test_positive_finite_values_do_not_change_any_other_runtime_field(self): + default=RuntimeConfig(**BOUNDS).model_dump() + tuned=RuntimeConfig(**BOUNDS,**dict(zip(POLL_FIELDS,TUNED))).model_dump() + for field,value in zip(POLL_FIELDS,TUNED): + self.assertEqual(tuned.pop(field),value) + default.pop(field) + self.assertEqual(tuned,default) + + def test_nonpositive_nonfinite_and_non_numeric_values_are_rejected(self): + for field in POLL_FIELDS: + for value in (0.,-0.,-0.01,float('nan'),float('inf'),float('-inf'),True,False,'invalid',None): + with self.subTest(field=field,value=value),self.assertRaises(ValidationError): + RuntimeConfig(**{field:value}) + + def test_actual_yaml_loader_and_dotted_overrides_revalidate_polling(self): + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'run.yaml' + path.write_text(yaml.safe_dump(payload(tmp,'consumer',polling=TUNED))) + cfg=load_config(str(path),['runtime.consumer_ref_poll_s=0.023']) + self.assertEqual(cfg.runtime.producer_backpressure_poll_s,TUNED[0]) + self.assertEqual(cfg.runtime.consumer_ref_poll_s,0.023) + self.assertEqual(cfg.runtime.ref_distributor_poll_s,TUNED[2]) + for field in POLL_FIELDS: + for value in ('0.0','-0.01','nan','inf','-inf','true'): + with self.subTest(field=field,value=value),self.assertRaises(ValidationError): + load_config(str(path),[f'runtime.{field}={value}']) + + +class RoleFactoryWiringTests(unittest.TestCase): + def test_producer_config_reaches_existing_builder_parameter_with_bounds_unchanged(self): + for polling,expected in ((None,DEFAULTS),(TUNED,TUNED)): + with self.subTest(polling=polling),tempfile.TemporaryDirectory() as tmp: + cfg=Config.model_validate(payload(tmp,'producer',polling=polling)) + with ( + mock.patch.dict(os.environ,{'DISAGG_REF_CHANNEL':str(Path(tmp)/'refs'), + 'DISAGG_RESIDENT_HIGH_WATERMARK_BYTES':'', + 'DISAGG_RESIDENT_LOW_WATERMARK_BYTES':''}), + mock.patch('specforge.runtime.data_plane.streaming_ref_channel.StreamingRefChannel'), + mock.patch('specforge.training.disaggregated._mooncake_store'), + mock.patch('specforge.training.assembly._load_input_tools'), + mock.patch('specforge.training.model_loading.resolve_draft_config'), + mock.patch('specforge.training.disaggregated._producer_capture_metadata',return_value=([1,2],8,32,32)), + mock.patch('specforge.inference.adapters.server_capture.SGLangServerCaptureAdapter'), + mock.patch('specforge.launch.build_disagg_online_producer',return_value=([],mock.Mock())) as build, + ): + _build_online(cfg,algorithm=ALGORITHM,build_model_bundle=mock.Mock(), + prepare_prompts=lambda *a,**k:[{'task_id':'p0','payload':{'input_ids':[1,2,3]}}], + optimizer_factory=mock.Mock(),logger=None) + args=build.call_args.kwargs + self.assertEqual(args['backpressure_poll_s'],expected[0]) + for name in ('producer_concurrency','in_flight_high_watermark','in_flight_low_watermark', + 'resident_high_watermark_bytes','resident_low_watermark_bytes', + 'feature_store_max_resident_bytes'): + self.assertEqual(args[name],BOUNDS[name]) + self.assertEqual(args['lease'],1) + self.assertEqual(args['prompt_seed'],42) + + def test_consumer_config_reaches_builder_without_changing_ack_or_math(self): + for polling,expected in ((None,DEFAULTS),(TUNED,TUNED)): + with self.subTest(polling=polling),tempfile.TemporaryDirectory() as tmp: + cfg=Config.model_validate(payload(tmp,'consumer',polling=polling)) + bundle=SimpleNamespace(model=object(),target_head=None,strategy_kwargs={}) + with ( + mock.patch.dict(os.environ,{'DISAGG_REF_CHANNEL':str(Path(tmp)/'refs')}), + mock.patch('specforge.runtime.data_plane.streaming_ref_channel.StreamingRefChannel'), + mock.patch('specforge.training.disaggregated._mooncake_store'), + mock.patch('specforge.launch.build_disagg_online_consumer',return_value=SimpleNamespace()) as build, + ): + _build_online(cfg,algorithm=ALGORITHM,build_model_bundle=lambda c:bundle, + prepare_prompts=mock.Mock(),optimizer_factory=mock.Mock(),logger=None) + args=build.call_args.kwargs + self.assertEqual(args['ref_poll_s'],expected[1]) + self.assertEqual(args['distributor_poll_s'],expected[2]) + self.assertIs(args['draft_model'],bundle.model) + self.assertEqual(args['async_ack'],False) + self.assertEqual(args['dispatch_num_anchors'],8) + self.assertEqual(args['dataloader_num_workers'],0) + self.assertEqual(args['max_steps'],1) + + +class EffectiveConsumerPollingTests(unittest.TestCase): + def build(self,root,**polling): + channel=StreamingRefChannel(str(root/'refs.jsonl')) + store=SimpleNamespace(retain_on_release=True) + captured={} + def assemble(**kwargs): + captured.update(kwargs) + return SimpleNamespace() + with ( + mock.patch('torch.distributed.is_initialized',return_value=False), + mock.patch.object(RefDistributor,'start',autospec=True,side_effect=lambda self:self) as start, + mock.patch('specforge.launch._assemble_trainer',side_effect=assemble), + ): + trainer=build_disagg_online_consumer(algorithm=ALGORITHM,feature_store=store,channel=channel, + draft_model=object(),optimizer_factory=object(),run_id='polling-test',output_dir=str(root/'out'), + collate_fn=lambda features:{},metadata_store=InMemoryMetadataStore(), + inbox_dir=str(root/'inboxes'),async_ack=False,idle_timeout_s=12.5,**polling) + self.assertEqual(start.call_count,1) + return trainer,captured + + def test_existing_direct_call_keeps_real_queue_and_distributor_defaults(self): + with tempfile.TemporaryDirectory() as tmp: + trainer,captured=self.build(Path(tmp)) + self.assertEqual(captured['ref_source']['queue'].poll_s,0.1) + self.assertEqual(trainer.ref_distributor.poll_s,0.05) + self.assertEqual(captured['ref_source']['queue'].idle_timeout_s,12.5) + self.assertEqual(trainer.ref_distributor.idle_timeout_s,12.5) + + def test_custom_builder_values_reach_real_constructor_instances(self): + with tempfile.TemporaryDirectory() as tmp: + trainer,captured=self.build(Path(tmp),ref_poll_s=TUNED[1],distributor_poll_s=TUNED[2]) + self.assertEqual(captured['ref_source']['queue'].poll_s,TUNED[1]) + self.assertEqual(trainer.ref_distributor.poll_s,TUNED[2]) + self.assertTrue(captured['ref_source']['defer_ack_until_durable']) + self.assertEqual(captured['async_ack'],False) + self.assertEqual(trainer.ref_distributor.refs_per_rank_step,1) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_data/test_prompt_length_filter.py b/tests/test_data/test_prompt_length_filter.py new file mode 100644 index 0000000000..e4c2f9719b --- /dev/null +++ b/tests/test_data/test_prompt_length_filter.py @@ -0,0 +1,121 @@ +"""Whole-record filtering at the native loader's public boundary.""" + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from pydantic import ValidationError + +from specforge.config.schema import DataConfig +from specforge.data.loss_mask import has_consecutive_supervised_tokens +from specforge.data.prompt_builder import prepare_prompt_tasks + + +class TestPromptLengthFilter(unittest.TestCase): + def load(self, records, **options): + report = {} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp, "data.jsonl") + path.write_text("".join(json.dumps(r) + "\n" for r in records)) + with contextlib.redirect_stdout(io.StringIO()) as output: + prompts = prepare_prompt_tasks( + path, + tokenizer=None, + chat_template=None, + max_length=65536, + is_preformatted=False, + train_only_last_turn=False, + cache_dir=None, + cache_key=None, + num_proc=1, + min_loss_tokens=2, + loss_mask_filter=has_consecutive_supervised_tokens, + overlength_policy="drop", + filter_report=report, + **options, + ) + self.assertIn("[prompt-filter]", output.getvalue()) + return prompts, report + + def test_boundary_and_original_ids_masks_survive(self): + records = [ + dict(id=str(n), input_ids=list(range(n)), loss_mask=[0] * (n - 2) + [1, 1]) + for n in (65535, 65536, 65537) + ] + prompts, report = self.load(records) + self.assertEqual( + [p["payload"] for p in prompts], + [{k: r[k] for k in ("input_ids", "loss_mask")} for r in records[:2]], + ) + self.assertEqual(report["accepted"], 2) + self.assertEqual(report["rows_seen"], 3) + self.assertEqual(report["dropped_other"], 0) + self.assertEqual(report["dropped_overlength"], 1) + self.assertEqual(report["kept_input_tokens"], 131071) + self.assertEqual(report["kept_loss_tokens"], 4) + self.assertEqual(report["dropped_samples"][0]["id"], "65537") + + def test_limit_counts_accepted_after_filter(self): + records = [ + dict(id="long", input_ids=[1] * 65537, loss_mask=[1] * 65537), + dict(id="invalid_supervision", input_ids=[2, 3, 4], loss_mask=[1, 0, 1]), + dict(id="keep", input_ids=[[4, 5]], loss_mask=[[1, 1]]), + dict(id="unread", input_ids=[6, 7], loss_mask=[1, 1]), + ] + prompts, report = self.load(records, max_prompts=1) + self.assertEqual( + prompts, [{"payload": {"input_ids": [4, 5], "loss_mask": [1, 1]}}] + ) + self.assertEqual(report["rows_seen"], 3) + self.assertEqual(report["dropped_other"], 1) + self.assertEqual(report["dropped_overlength"], 1) + + def test_zero_and_none_limits_keep_all_accepted_records(self): + records = [ + dict(id="long", input_ids=[1] * 65537, loss_mask=[1] * 65537), + dict(id="first", input_ids=[2, 3], loss_mask=[1, 1]), + dict(id="second", input_ids=[4, 5, 6], loss_mask=[0, 1, 1]), + ] + for limit in (None, 0): + with self.subTest(limit=limit): + prompts, report = self.load(records, max_prompts=limit) + self.assertEqual( + [p["payload"]["input_ids"] for p in prompts], [[2, 3], [4, 5, 6]] + ) + self.assertEqual(report["rows_seen"], 3) + self.assertEqual(report["accepted"], 2) + self.assertEqual(report["dropped_overlength"], 1) + + def test_malformed_overlength_still_fails_validation(self): + with self.assertRaisesRegex(ValueError, "mismatched"): + self.load([dict(input_ids=[1] * 65537, loss_mask=[1, 1])]) + with self.assertRaisesRegex(ValueError, "must be 0 or 1"): + self.load([dict(input_ids=[1] * 65537, loss_mask=[1] * 65536 + [2])]) + + def test_raw_conversations_cannot_silently_truncate(self): + with self.assertRaisesRegex(ValueError, "requires pre-tokenized"): + self.load([{"conversations": [{"role": "user", "content": "hello"}]}]) + + def test_schema_requires_native_source_and_defaults_to_legacy(self): + self.assertEqual( + DataConfig(prompts_path="data.jsonl").overlength_policy, "truncate" + ) + self.assertEqual( + DataConfig( + prompts_path="data.jsonl", overlength_policy="drop" + ).overlength_policy, + "drop", + ) + with self.assertRaisesRegex( + ValidationError, "requires native data.prompts_path" + ): + DataConfig(train_data_path="raw.jsonl", overlength_policy="drop") + with self.assertRaises(ValidationError): + DataConfig(prompts_path="data.jsonl", overlength_policy="invalid") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_optimizer/test_torchspec_compat.py b/tests/test_optimizer/test_torchspec_compat.py new file mode 100644 index 0000000000..7919e8b4e4 --- /dev/null +++ b/tests/test_optimizer/test_torchspec_compat.py @@ -0,0 +1,42 @@ +import copy +import math +import unittest +from unittest.mock import patch +import torch +from specforge.config.schema import TrainingConfig +from specforge.optimizer import BF16Optimizer + +class TorchSpecCompatibility(unittest.TestCase): + def make(self): + model=torch.nn.Linear(2,2,bias=False).to(torch.bfloat16) + opt=BF16Optimizer(model,lr=1e-4,weight_decay=.01,min_lr=1e-5,total_steps=1197,warmup_ratio=.01,lr_scheduler='torchspec_cosine') + return model,opt + + def test_original_schedule_and_decay(self): + model,opt=self.make() + self.assertEqual(opt.optimizer.param_groups[0]['weight_decay'],.01) + self.assertEqual(opt.get_learning_rate(),0) + for step in range(1,1198): + model.weight.grad=torch.ones_like(model.weight) + opt.step() + expected=1e-4*step/11 if step<=11 else 1e-5+.5*(1e-4-1e-5)*(1+math.cos(math.pi*(step-11)/(1197-11))) + self.assertAlmostEqual(opt.get_learning_rate(),expected,places=14) + + def test_checkpoint_next_update(self): + model,opt=self.make() + for _ in range(12): + model.weight.grad=torch.ones_like(model.weight);opt.step() + other,restored=self.make() + other.load_state_dict(model.state_dict()) + with patch("specforge.optimizer.print_on_rank0"): + restored.load_state_dict(copy.deepcopy(opt.state_dict())) + model.weight.grad=torch.full_like(model.weight,.25) + other.weight.grad=model.weight.grad.clone() + opt.step();restored.step() + self.assertEqual(opt.get_learning_rate(),restored.get_learning_rate()) + self.assertTrue(torch.equal(model.weight,other.weight)) + for a,b in zip(opt.fp32_params,restored.fp32_params):self.assertTrue(torch.equal(a,b)) + + def test_config_rejects_invalid_min_lr(self): + for kw in [dict(learning_rate=1e-4,min_lr=.1),dict(lr_scheduler='constant',min_lr=1e-5)]: + with self.assertRaises(ValueError):TrainingConfig(**kw) diff --git a/tests/test_runtime/test_capture_begin_parallel.py b/tests/test_runtime/test_capture_begin_parallel.py new file mode 100644 index 0000000000..06953d66e3 --- /dev/null +++ b/tests/test_runtime/test_capture_begin_parallel.py @@ -0,0 +1,202 @@ +"""Begin/recovery singleflight is scoped to original identities, never the store.""" +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from unittest import mock + +from specforge.inference.capture_protocol import CaptureProtocol +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from tests.test_runtime.test_server_capture import _FakeMooncakeStore + + +class Transport: + def __init__(self): + self.calls = [] + self.gate = None + self.lock = threading.Lock() + self.lose_batch = False + + def __call__(self, url, *, json_body, timeout): + body = dict(json_body) + with self.lock: + self.calls.append((url, body)) + if self.gate: + self.gate(url, body) + tasks = body['tasks'] if body['action'] == 'begin_batch' else [body] + replies = [{**task, 'reservation_token': task['attempt_id'] + '-token', + **({'object_key_layout': 'attempt-v1'} if task.get('publication_ownership') == 'attempt-v1' else {})} + for task in tasks] + if body['action'] == 'begin_batch': + if self.lose_batch: + self.lose_batch = False + raise TimeoutError('lost batch reply after server reservation') + return {'attempts': replies} + return replies[0] + + +class ObservedMetadataLock: + def __init__(self, wrapped, observed): + self.wrapped, self.observed = wrapped, observed + + def __enter__(self): + self.wrapped.acquire() + return self + + def __exit__(self, *exc): + self.wrapped.release() + if threading.current_thread().name.startswith('recovery'): + self.observed.set() + + +class BeginParallelTests(unittest.TestCase): + def setUp(self): + self.transport = Transport() + self.store = MooncakeFeatureStore(store=_FakeMooncakeStore(), store_id='store') + self.cap = dict(server_epoch='epoch', begin_batch=True, published_ownership='attempt-v1') + self.first = CaptureProtocol('http://endpoint-1', self.transport, 1) + self.second = CaptureProtocol('http://endpoint-2', self.transport, 1) + + def request(self, sid, attempt=None): + return (dict(store_id='store', sample_id=sid, gen=1, features={'aux': 'hidden'}, passthrough=[]), + [1, 2, 3], attempt or sid + '-attempt', ['hidden']) + + def begin(self, protocol, sid): + return self.store.begin_capture_attempts(protocol, self.cap, [self.request(sid)])[0] + + def test_two_endpoints_enter_http_concurrently(self): + barrier = threading.Barrier(2, timeout=2) + self.transport.gate = lambda url, body: barrier.wait() + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(self.begin, self.first, 'first') + second = executor.submit(self.begin, self.second, 'second') + self.assertEqual(first.result(timeout=3)['attempt_id'], 'first-attempt') + self.assertEqual(second.result(timeout=3)['attempt_id'], 'second-attempt') + self.assertEqual(len(self.transport.calls), 2) + self.assertFalse(self.store._capture_pending_begin) + + def test_fresh_prepare_does_not_wait_for_other_inflight_http(self): + entered, release = threading.Event(), threading.Event() + def gate(url, body): + entered.set() + self.assertTrue(release.wait(2)) + self.transport.gate = gate + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(self.begin, self.first, 'blocked') + try: + self.assertTrue(entered.wait(1)) + executor.submit(self.store.prepare_capture_retry, 'unrelated').result(timeout=.5) + self.assertFalse(first.done()) + finally: + release.set() + first.result(timeout=2) + + def test_same_sid_cross_endpoint_cannot_replace_pending_or_active_attempt(self): + entered, release = threading.Event(), threading.Event() + def gate(url, body): + entered.set() + self.assertTrue(release.wait(2)) + self.transport.gate = gate + with ThreadPoolExecutor(max_workers=1) as executor: + first = executor.submit(self.begin, self.first, 'same') + try: + self.assertTrue(entered.wait(1)) + original = self.store._capture_pending_begin['same'] + with self.assertRaisesRegex(RuntimeError, 'pending'): + self.begin(self.second, 'same') + self.assertIs(self.store._capture_pending_begin['same'], original) + self.assertEqual(len(self.transport.calls), 1) + finally: + release.set() + first.result(timeout=2) + with self.assertRaisesRegex(RuntimeError, 'prior capture'): + self.begin(self.second, 'same') + self.assertEqual(len(self.transport.calls), 1) + + def test_waiting_recovery_does_not_replay_completed_original_flight(self): + entered, release, snapshot = threading.Event(), threading.Event(), threading.Event() + def gate(url, body): + entered.set() + self.assertTrue(release.wait(2)) + self.transport.gate = gate + self.store._lock = ObservedMetadataLock(self.store._lock, snapshot) + with ThreadPoolExecutor(max_workers=1) as original_executor, ThreadPoolExecutor(max_workers=1, thread_name_prefix='recovery') as recovery_executor: + original = original_executor.submit(self.begin, self.first, 'same') + try: + self.assertTrue(entered.wait(1)) + recovery = recovery_executor.submit(self.store._recover_capture_begin, 'same') + self.assertTrue(snapshot.wait(1)) + finally: + release.set() + original.result(timeout=2) + self.assertIsNone(recovery.result(timeout=2)) + self.assertEqual(len(self.transport.calls), 1) + + def test_waiting_recovery_never_retargets_replacement_intent(self): + self.transport.lose_batch = True + with self.assertRaises(TimeoutError): + self.begin(self.first, 'same') + original = self.store._capture_pending_begin['same'] + snapshot = threading.Event() + self.store._lock = ObservedMetadataLock(self.store._lock, snapshot) + replacement = replace(original, protocol=self.second, capability=dict(self.cap, server_epoch='new-epoch'), + attempt_id='replacement', flight=threading.RLock()) + with original.flight: + with ThreadPoolExecutor(max_workers=1, thread_name_prefix='recovery') as executor: + recovery = executor.submit(self.store._recover_capture_begin, 'same') + self.assertTrue(snapshot.wait(1)) + with self.store._lock: + self.store._capture_pending_begin['same'] = replacement + # The executor must not join while this thread owns its flight. + original.flight.release() + try: + self.assertIsNone(recovery.result(timeout=2)) + finally: + original.flight.acquire() + self.assertIs(self.store._capture_pending_begin['same'], replacement) + self.assertEqual(len(self.transport.calls), 1) + + def test_partial_track_failure_preserves_only_original_unfinished_intent(self): + original_track = self.store.track_external_attempt + failures = [True] + def track(sid, **kwargs): + if sid == 'second' and failures[0]: + failures[0] = False + raise RuntimeError('metadata tracking failed') + return original_track(sid, **kwargs) + with mock.patch.object(self.store, 'track_external_attempt', side_effect=track): + with self.assertRaisesRegex(RuntimeError, 'tracking failed'): + self.store.begin_capture_attempts(self.first, self.cap, [self.request('first'), self.request('second')]) + self.assertNotIn('first', self.store._capture_pending_begin) + intent = self.store._capture_pending_begin['second'] + wire = self.store._recover_capture_begin('second') + self.assertEqual(wire['reservation_token'], 'second-attempt-token') + self.assertEqual(intent.protocol.url, self.first.url) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual([body['action'] for _, body in self.transport.calls], ['begin_batch', 'begin']) + + def test_lost_batch_recovery_retains_original_endpoint_epoch_and_attempts(self): + self.transport.lose_batch = True + with self.assertRaises(TimeoutError): + self.store.begin_capture_attempts(self.first, self.cap, [self.request('first'), self.request('second')]) + intents = dict(self.store._capture_pending_begin) + self.store._recover_capture_begin('second') + self.store._recover_capture_begin('first') + for url, body in self.transport.calls[1:]: + self.assertEqual(url, self.first.url) + self.assertEqual(body['server_epoch'], 'epoch') + self.assertEqual(body['attempt_id'], intents[body['sample_id']].attempt_id) + self.assertFalse(self.store._capture_pending_begin) + + def test_legacy_single_begin_fallback_reenters_batch_flight(self): + self.cap.pop('begin_batch') + with ThreadPoolExecutor(max_workers=1) as executor: + replies = executor.submit(self.store.begin_capture_attempts, self.first, self.cap, + [self.request('first'), self.request('second')]).result(timeout=2) + self.assertEqual(len(replies), 2) + self.assertEqual([body['action'] for _, body in self.transport.calls], ['begin', 'begin']) + self.assertFalse(self.store._capture_pending_begin) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_capture_fencing.py b/tests/test_runtime/test_capture_fencing.py new file mode 100644 index 0000000000..97f047ee8d --- /dev/null +++ b/tests/test_runtime/test_capture_fencing.py @@ -0,0 +1,253 @@ +"""Attempt-fence integration through real adapter and separate FeatureStores.""" +import uuid +import unittest +from unittest import mock + +from specforge.inference.adapters.server_capture import SGLangServerCaptureAdapter as _NativeAdapter +from specforge.inference.capture_protocol import CaptureProtocol + +def SGLangServerCaptureAdapter(*args, **kwargs): + adapter = _NativeAdapter(*args, **kwargs) + adapter._capture_protocol = CaptureProtocol(adapter.base_url, kwargs["control_fn"], adapter.timeout_s) + return adapter +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from tests.test_runtime.test_server_capture import ( + _FakeMooncakeStore, _StubCaptureServer, _capture_schema, _task, _eagle3_contract, + HIDDEN, AUX_LAYERS, _request_specs, +) + + +class ProtocolServer: + def __init__(self, backend): + self.backend = backend + self.stub = _StubCaptureServer(backend) + self.stub._result = lambda result: result + self.active = {} + self.retired = {} + self.drained = True + self.events = [] + self.capability = dict(protocol_version=1, server_epoch='epoch', capture_namespace='namespace', + capture_signature='signature', cache_enabled=True, + schema=[dict(name='aux', width=HIDDEN*len(AUX_LAYERS), dtype='float32', layer_ids=list(AUX_LAYERS)), + dict(name='last_hidden', width=HIDDEN, dtype='float32', layer_ids=[])]) + + def __call__(self, url, json_body, timeout): + if url.endswith('/spec_capture/control'): + body = json_body + action = body['action'] + self.events.append(action) + if action == 'capability': + return self.capability + sid = body['sample_id'] + if action == 'begin': + if sid in self.active: + old = self.active[sid] + if old['attempt_id'] == body['attempt_id'] and old['request_digest'] == body['request_digest']: + return dict(old) + raise RuntimeError('prior attempt unclean') + entry = {k: body[k] for k in ('store_id','sample_id','gen','attempt_id','server_epoch')} + entry.update(reservation_token=uuid.uuid4().hex, state='RESERVED', writer_drained=False, + request_digest=body['request_digest'], cleanup_owner=None, cleanup_token=None, readers={}, released_readers=set()) + self.active[sid] = entry + return dict(entry) + entry = self.active.get(sid) + if entry is None or entry['reservation_token'] != body['reservation_token']: + entry = self.retired.get(body['reservation_token']) + if entry is None or entry['reservation_token'] != body['reservation_token']: + raise RuntimeError('unknown attempt') + if action == 'resolve': + if entry['state'] == 'RETIRED': + return dict(entry) + entry['writer_drained'] = self.drained + if body.get('cancel'): + entry['state'] = 'WRITER_DRAINED' if self.drained else 'DRAINING' + return dict(entry) + if action == 'read_acquire': + if entry['state'] != 'PUBLISHED' or entry['cleanup_token']: + raise RuntimeError('attempt is not readable') + owner = body['reader_owner'] + token = entry['readers'].setdefault(owner, uuid.uuid4().hex) + return dict(entry,reader_owner=owner,reader_token=token) + if action == 'read_release': + owner,token = body['reader_owner'],body['reader_token'] + if entry['readers'].get(owner) == token: + entry['readers'].pop(owner) + entry['released_readers'].add((owner,token)) + elif (owner,token) not in entry['released_readers']: + raise RuntimeError('unknown read permit') + return dict(entry,reader_owner=owner,reader_token=token,reader_released=True) + if action == 'cleanup_acquire': + if entry['readers']: + raise RuntimeError('active readers') + if entry['state'] == 'RETIRED': + return dict(entry) + if not entry['writer_drained']: + raise RuntimeError('writer still live') + if entry['cleanup_owner'] not in (None, body['cleanup_owner']): + raise RuntimeError('cleanup owned by another store') + entry['cleanup_owner'] = body['cleanup_owner'] + entry['cleanup_token'] = entry['cleanup_token'] or uuid.uuid4().hex + return dict(entry) + if action == 'cleanup_done': + if body.get('cleanup_token') != entry['cleanup_token']: + raise RuntimeError('cleanup token mismatch') + if not entry['writer_drained']: + raise RuntimeError('writer still live') + if any(key.startswith(f"{entry['store_id']}/{sid}/g1/") for key in self.backend._d): + raise RuntimeError('cleanup ack before key removal') + entry['state'] = 'RETIRED' + self.retired[entry['reservation_token']] = entry + self.active.pop(sid, None) + return dict(entry) + raise RuntimeError(action) + rows = self.stub(url, json_body, timeout) + for row, spec, tokens in zip(rows, _request_specs(json_body), json_body['input_ids']): + entry = self.active[spec['sample_id']] + entry.update(state='PUBLISHED', writer_drained=True) + row['meta_info']['spec_capture'].update(entry) + row['meta_info']['spec_capture'].update({k:self.capability[k] for k in ('protocol_version','capture_namespace','capture_signature')}) + row['meta_info']['spec_capture'].update(H=0, L=0, N=len(tokens), span=[0,len(tokens)], execution_epoch=0, reused_tokens=0) + return rows + + +class FenceTests(unittest.TestCase): + def setUp(self): + self.backend = _FakeMooncakeStore() + self.server = ProtocolServer(self.backend) + self.producer = MooncakeFeatureStore(store=self.backend, store_id='run0') + self.adapter = SGLangServerCaptureAdapter('http://server', self.producer, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, + timeout_s=0.01) + + def produce(self, index=0): + return self.adapter.produce_refs([_task(index, 5)], capture=_eagle3_contract())[0] + + def test_separate_consumer_release_carries_fence_and_retires_registry(self): + ref = self.produce() + self.assertIn('capture_fence', ref.metadata) + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', self.server): + tensors, handle = consumer.get(ref) + self.assertEqual(tensors['hidden_state'].shape[1], 5) + consumer.release(handle) + self.assertFalse(self.backend._d) + self.assertFalse(self.server.active) + self.assertIn('cleanup_done', self.server.events) + with mock.patch('specforge.inference.capture_protocol.http_control', self.server): + with self.assertRaises(KeyError): + consumer.get(ref) + + def test_timeout_retains_keys_until_writer_drain(self): + payload = self.adapter._spec_capture_payload(_task(0,5)) + wire = self.adapter._capture_protocol.begin(self.server.capability, payload, [1]*5, 'first') + self.producer.track_capture_fence(wire, ['hidden_state'], control_fn=self.server) + self.producer.track_external_attempt(payload['sample_id'], generation=1, feature_names=['hidden_state']) + self.backend._d[f"run0/{payload['sample_id']}/g1/hidden_state"] = b'old' + self.server.drained = False + with self.assertRaises(TimeoutError): + self.producer.prepare_capture_retry(payload['sample_id']) + self.assertTrue(self.backend._d) + self.assertTrue(self.producer._external_provisional) + self.server.drained = True + self.producer.prepare_capture_retry(payload['sample_id']) + self.assertFalse(self.backend._d) + self.assertFalse(self.producer._capture_fences) + + def test_old_cleanup_and_duplicate_release_cannot_delete_new_g1(self): + ref = self.produce() + old_token = ref.metadata['capture_fence']['reservation_token'] + values, handle = self.producer.get(ref) + self.producer.release(handle) + new_ref = self.produce() + new_token = new_ref.metadata['capture_fence']['reservation_token'] + self.assertNotEqual(old_token,new_token) + before = dict(self.backend._d) + self.producer.abort(ref.sample_id, attempt_token=old_token) + self.producer.release(handle) + self.assertEqual(before,self.backend._d) + with self.assertRaises(KeyError): + self.producer.get(ref) + values, new_handle = self.producer.get(new_ref) + self.producer.release(new_handle) + self.assertFalse(self.backend._d) + + def test_old_producer_gc_cannot_remove_new_cross_process_g1(self): + ref = self.produce() + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', self.server): + _, handle = consumer.get(ref) + consumer.release(handle) + replacement_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + replacement_adapter = SGLangServerCaptureAdapter('http://server', replacement_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server) + replacement_adapter.produce_refs([_task(0,5)], capture=_eagle3_contract()) + before = dict(self.backend._d) + self.producer.max_hold_age_s = 0 + self.producer.gc(now=10**12) + self.assertEqual(before, self.backend._d) + self.assertFalse(self.producer._capture_fences) + + def test_lost_begin_response_recovers_same_intent_and_cleans(self): + original = self.adapter._capture_protocol.post + lost = [False] + def transport(url,json_body,timeout): + reply = original(url,json_body,timeout) + if json_body.get('action') == 'begin' and not lost[0]: + lost[0] = True + raise TimeoutError('begin response lost after reservation') + return reply + self.adapter._capture_protocol.post = transport + with self.assertRaises(TimeoutError): + self.produce() + self.assertTrue(self.producer._capture_pending_begin) + self.assertTrue(self.server.active) + self.producer.discard_external_attempts() + self.assertFalse(self.producer._capture_pending_begin) + self.assertFalse(self.server.active) + self.assertFalse(self.producer._capture_fences) + + def test_lost_read_acquire_reply_recovers_intent_before_cleanup(self): + ref=self.produce() + fence=self.producer._capture_fences[ref.sample_id] + original=fence.post + lost=[True] + def transport(url,json_body,timeout): + reply=original(url,json_body,timeout) + if json_body.get('action')=='read_acquire' and lost[0]: + lost[0]=False + raise TimeoutError('read acquire reply lost') + return reply + fence.post=transport + with self.assertRaises(TimeoutError): + self.producer.get(ref) + self.assertEqual(len(self.server.active[ref.sample_id]['readers']),1) + self.assertEqual(self.producer.health()['capture_read_intents'],1) + self.producer.drain_pending_removals(max_attempts=2,retry_interval_s=0) + self.assertEqual(len(self.server.active[ref.sample_id]['readers']),0) + self.assertEqual(self.producer.health()['capture_read_intents'],0) + _,handle=self.producer.get(ref) + self.producer.release(handle) + + def test_live_lease_prevents_same_name_retry(self): + ref = self.produce() + _, handle = self.producer.get(ref) + with self.assertRaisesRegex(RuntimeError,'active consumer'): + self.producer.prepare_capture_retry(ref.sample_id) + self.producer.release(handle) + + def test_negotiated_identity_mismatch_fails_without_adopting(self): + original = self.adapter.post_fn + def corrupt(url,json_body,timeout): + result = original(url,json_body,timeout) + result[0]['meta_info']['spec_capture']['server_epoch'] = 'wrong' + return result + self.adapter.post_fn = corrupt + with self.assertRaisesRegex(RuntimeError,'identity mismatch'): + self.produce() + self.assertTrue(self.producer._external_provisional) + self.producer.discard_external_attempts() + self.assertFalse(self.backend._d) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_capture_producer_coalescing.py b/tests/test_runtime/test_capture_producer_coalescing.py new file mode 100644 index 0000000000..9c660b1dfd --- /dev/null +++ b/tests/test_runtime/test_capture_producer_coalescing.py @@ -0,0 +1,513 @@ +"""Negotiated lease-one producer batching and independent completion contracts.""" +import copy +import os +import tempfile +import threading +import time +import unittest +from concurrent.futures import Future, ThreadPoolExecutor +from unittest import mock + +from specforge.inference.capture_protocol import CaptureProtocol +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from tests.test_runtime.test_capture_fencing import ProtocolServer +from tests.test_runtime.test_disagg_multiserver import _adapter, _build, _prompts +from tests.test_runtime.test_server_capture import _FakeMooncakeStore, _request_specs + + +class BatchServer(ProtocolServer): + def __init__(self, backend): + super().__init__(backend) + self.capability['begin_batch'] = True + for item in self.capability['schema']: + item['dtype'] = 'bfloat16' + self.batches = [] + self.generates = [] + self.gate = None + self.lock = threading.RLock() + + def __call__(self, url, json_body, timeout): + body = copy.deepcopy(json_body) + action = body.get('action', 'generate') + if action == 'begin_batch': + with self.lock: + self.batches.append(body) + replies = [super(BatchServer, self).__call__(url, dict(task, action='begin'), timeout) + for task in body['tasks']] + if self.gate: + self.gate(action, body) + return {'attempts': replies} + if action == 'generate': + with self.lock: + self.generates.append(body) + if self.gate: + self.gate(action, body) + with self.lock: + return super().__call__(url, body, timeout) + + +def wait_for(predicate, timeout=3): + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + raise AssertionError('condition did not become true') + time.sleep(.001) + + +class CoalescedProducerTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.backend = _FakeMooncakeStore() + self.store = MooncakeFeatureStore(store=self.backend, store_id='run0') + self.server = BatchServer(self.backend) + self.channel = StreamingRefChannel(os.path.join(self.directory.name, 'refs')) + + def build(self, n=8, concurrency=4, servers=None, **kwargs): + adapters = [] + for i, server in enumerate(servers or [self.server]): + adapter = _adapter(self.store, server, url=f'http://server{i}') + adapter._capture_protocol = CaptureProtocol(adapter.base_url, server, 1) + adapters.append(adapter) + lease = kwargs.pop('lease', 1) + return _build(adapters, _prompts(n), self.store, self.channel, + lease=lease, producer_concurrency=concurrency, **kwargs) + + def drive_async(self, drive, **kwargs): + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(drive, **kwargs) + self.addCleanup(pool.shutdown) + return future + + def test_initial_idle_slots_share_begin_but_generate_independently(self): + workers, drive = self.build(n=4) + self.assertEqual(drive(), 4) + self.assertEqual([len(b['tasks']) for b in self.server.batches], [4]) + self.assertEqual([len(b['input_ids']) for b in self.server.generates], [1]*4) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_concurrency_twelve_initial_burst_is_one_registration(self): + workers, drive = self.build(n=12, concurrency=12) + self.assertEqual(drive(), 12) + self.assertEqual([len(b['tasks']) for b in self.server.batches], [12]) + self.assertEqual([len(b['input_ids']) for b in self.server.generates], [1]*12) + self.assertEqual(workers[0].health()['in_flight'], 0) + + def test_returned_output_cannot_silently_exceed_reserved_schema(self): + for item in self.server.capability['schema']: + item['dtype'] = 'int8' # Deliberately false server capability. + workers, drive = self.build(n=2, concurrency=2, max_worker_failures=1) + with self.assertRaisesRegex(RuntimeError, 'dead workers'): + drive() + self.assertIn('exceeded negotiated byte reservation', self.channel.failure()) + self.assertEqual(self.channel.published, 0) + self.assertFalse(self.backend._d) + self.assertFalse(self.server.active) + self.assertEqual(workers[0].health()['in_flight'], 0) + + def test_done_generate_publishes_while_followup_begin_is_blocked(self): + release_slow, begin_blocked, release_begin = (threading.Event() for _ in range(3)) + slow = {} + def gate(action, body): + if action == 'begin_batch' and len(self.server.batches) == 2: + begin_blocked.set() + self.assertTrue(release_begin.wait(3)) + if action == 'generate': + sid = _request_specs(body)[0]['sample_id'] + if sid == slow['sid']: + self.assertTrue(release_slow.wait(3)) + original = self.server.__call__ + # Capture the initial batch identity before any generate can execute. + def transport(url, json_body, timeout): + if json_body.get('action') == 'begin_batch' and not slow: + slow['sid'] = json_body['tasks'][0]['sample_id'] + return original(url, json_body, timeout) + self.server.gate = gate + workers, drive = self.build(n=7, concurrency=3, servers=[transport]) + future = self.drive_async(drive) + try: + self.assertTrue(begin_blocked.wait(2)) + release_slow.set() + wait_for(lambda: self.channel.published >= 3) + self.assertFalse(future.done()) + self.assertFalse(release_begin.is_set()) + finally: + release_slow.set() + release_begin.set() + self.assertEqual(future.result(timeout=3), 7) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertGreater(len(self.server.batches), 1) + + def test_late_idle_slots_coalesce_after_inflight_begin(self): + second_begin = threading.Event() + release_begin = threading.Event() + release_initial = threading.Event() + initial = [] + original = self.server.__call__ + def transport(url, json_body, timeout): + if json_body.get('action') == 'begin_batch' and not initial: + initial.extend(task['sample_id'] for task in json_body['tasks']) + return original(url, json_body, timeout) + def gate(action, body): + if action == 'begin_batch' and len(self.server.batches) == 2: + second_begin.set() + self.assertTrue(release_begin.wait(3)) + if action == 'generate': + sid = _request_specs(body)[0]['sample_id'] + if sid in initial[1:]: + self.assertTrue(release_initial.wait(3)) + self.server.gate = gate + workers, drive = self.build(n=9, concurrency=4, servers=[transport]) + source = workers[0].feature_source + prepared_late = set() + prepare = source.prepare_refs_batches + def observe_prepare(batches, *, capture): + result = prepare(batches, capture=capture) + if second_begin.is_set(): + prepared_late.update(task.task_id for tasks in batches for task in tasks) + return result + source.prepare_refs_batches = observe_prepare + future = self.drive_async(drive) + try: + self.assertTrue(second_begin.wait(2)) + release_initial.set() + wait_for(lambda: len(prepared_late) == 3) + finally: + release_initial.set() + release_begin.set() + self.assertEqual(future.result(timeout=3), 9) + self.assertEqual([len(b['tasks']) for b in self.server.batches[:3]], [4, 1, 3]) + + def test_other_endpoint_registers_while_first_endpoint_is_blocked(self): + entered, other_entered, release = (threading.Event() for _ in range(3)) + other = BatchServer(self.backend) + def first_gate(action, body): + if action == 'begin_batch': + entered.set() + self.assertTrue(release.wait(3)) + def other_gate(action, body): + if action == 'begin_batch': + self.assertTrue(entered.wait(2)) + other_entered.set() + self.server.gate, other.gate = first_gate, other_gate + _, drive = self.build(n=8, concurrency=2, servers=[self.server, other]) + future = self.drive_async(drive) + try: + self.assertTrue(other_entered.wait(2)) + finally: + release.set() + self.assertEqual(future.result(timeout=3), 8) + + def test_generate_submit_failure_retires_registered_preleases(self): + workers, drive = self.build(n=4) + submit = ThreadPoolExecutor.submit + def guarded_submit(executor, fn, *args, **kwargs): + if getattr(fn, '__name__', '') == 'run_leased': + raise RuntimeError('injected generate submit failure') + return submit(executor, fn, *args, **kwargs) + with mock.patch.object(ThreadPoolExecutor, 'submit', guarded_submit): + with self.assertRaisesRegex(RuntimeError, 'generate submit failure'): + drive() + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + self.assertIsNotNone(self.channel.failure()) + + def test_cancelled_generate_future_releases_prelease_and_fence(self): + workers, drive = self.build(n=4, max_worker_failures=1) + submit = ThreadPoolExecutor.submit + def cancelling_submit(executor, fn, *args, **kwargs): + if getattr(fn, '__name__', '') == 'run_leased': + future = Future() + future.cancel() + return future + return submit(executor, fn, *args, **kwargs) + with mock.patch.object(ThreadPoolExecutor, 'submit', cancelling_submit): + with self.assertRaisesRegex(RuntimeError, 'dead workers'): + drive() + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_stale_prepared_cleanup_preserves_peer_replacement_attempt(self): + workers, _drive = self.build(n=1, concurrency=2) + source = workers[0].feature_source + # Use real task leases and store fences. + workers[0].start() + workers[0].controller.ingest_prompts(_prompts(1)) + tasks = workers[0].lease_tasks(1) + old = source.prepare_refs_batches([tasks], capture=workers[0].capture) + source.register_prepared_batches(old) + sid = old[0]['payloads'][0]['sample_id'] + self.store.prepare_capture_retry(sid) # A peer retires the old writer. + new = source.prepare_refs_batches([tasks], capture=workers[0].capture) + source.register_prepared_batches(new) + fence = self.store._capture_fences[sid] + source.discard_prepared_batches(old) + self.assertIs(self.store._capture_fences[sid], fence) + self.assertEqual(self.server.active[sid]['reservation_token'], fence.token) + source.discard_prepared_batches(new) + workers[0].fail_leased(tasks, 'test-cleanup', unhealthy=False) + self.assertFalse(self.server.active) + + def test_unknown_old_cleanup_never_recovers_replacement_pending_intent(self): + workers, _drive = self.build(n=1, concurrency=2) + source = workers[0].feature_source + workers[0].start() + workers[0].controller.ingest_prompts(_prompts(1)) + tasks = workers[0].lease_tasks(1) + calls = [] + original = source._capture_protocol.post + def lose_batch(url, json_body, timeout): + calls.append(copy.deepcopy(json_body)) + reply = original(url, json_body, timeout) + if json_body.get('action') == 'begin_batch': + raise TimeoutError('lost begin response') + return reply + source._capture_protocol.post = lose_batch + old = source.prepare_refs_batches([tasks], capture=workers[0].capture) + with self.assertRaises(TimeoutError): + source.register_prepared_batches(old) + sid = old[0]['payloads'][0]['sample_id'] + self.store.prepare_capture_retry(sid) + new = source.prepare_refs_batches([tasks], capture=workers[0].capture) + with self.assertRaises(TimeoutError): + source.register_prepared_batches(new) + pending = self.store._capture_pending_begin[sid] + calls_before = len(calls) + source.discard_prepared_batches(old) + self.assertEqual(len(calls), calls_before) + self.assertIs(self.store._capture_pending_begin[sid], pending) + self.assertEqual(self.server.active[sid]['attempt_id'], pending.attempt_id) + source.discard_prepared_batches(new) + workers[0].fail_leased(tasks, 'test-cleanup', unhealthy=False) + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + + def test_unknown_begin_reply_recovers_original_intents_before_retry(self): + calls = [] + original = self.server.__call__ + lost = [False] + def transport(url, json_body, timeout): + calls.append(copy.deepcopy(json_body)) + reply = original(url, json_body, timeout) + if json_body.get('action') == 'begin_batch' and not lost[0]: + lost[0] = True + raise TimeoutError('lost begin reply') + return reply + workers, drive = self.build(n=4, servers=[transport]) + self.assertEqual(drive(), 4) + first = next(body for body in calls if body.get('action') == 'begin_batch') + recovered = [body for body in calls if body.get('action') == 'begin'] + self.assertEqual({b['attempt_id'] for b in recovered}, {b['attempt_id'] for b in first['tasks']}) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + + def test_http_409_refresh_never_retargets_original_begin_intents(self): + import requests + calls = [] + original = self.server.__call__ + rejected = [False] + recovered = [] + def transport(url, json_body, timeout): + calls.append(copy.deepcopy(json_body)) + if json_body.get('action') == 'begin_batch' and not rejected[0]: + # A response encoding failure may surface as HTTP 409 after + # reservations were granted. It is not proof of rollback. + original(url, json_body, timeout) + rejected[0] = True + self.server.capability['server_epoch'] = 'refreshed-epoch' + response = requests.Response() + response.status_code = 409 + raise requests.HTTPError('ambiguous server rejection', response=response) + if json_body.get('action') == 'begin': + recovered.append(copy.deepcopy(json_body)) + # Cleanup must recover the exact old reservation despite the + # refreshed discovery; reject any new identity immediately. + self.assertEqual(json_body['server_epoch'], 'epoch') + return original(url, json_body, timeout) + workers, drive = self.build(n=4, servers=[transport], max_worker_failures=1) + with self.assertRaisesRegex(RuntimeError, 'dead workers'): + drive() + batch = next(body for body in calls if body.get('action') == 'begin_batch') + self.assertEqual({body['attempt_id'] for body in recovered}, + {body['attempt_id'] for body in batch['tasks']}) + self.assertEqual(len([body for body in calls if body.get('action') == 'begin_batch']), 1) + self.assertFalse(self.server.generates) + self.assertFalse(self.store._capture_pending_begin) + self.assertFalse(self.server.active) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_unreturned_outputs_consume_global_hard_cap_credit(self): + # Both negotiated bfloat16 engine tensors plus two int64 passthrough tensors. + per_token = sum(item['width'] * 2 for item in self.server.capability['schema']) + 16 + cap = per_token * 42 # Fits two longest prompts across both endpoints. + other = BatchServer(self.backend) + _, drive = self.build(n=16, concurrency=8, servers=[self.server, other], + feature_store_max_resident_bytes=cap) + reader = StreamingRefChannel(self.channel.path) + outstanding = 0 + peak = 0 + lock = threading.Lock() + def gate(action, body): + nonlocal outstanding, peak + if action == 'begin_batch': + # Derive output lengths from request ids in this fixed fixture. + size = sum((6 + int(task['sample_id'].split(':p')[1])) * per_token for task in body['tasks']) + with lock: + outstanding += size + peak = max(peak, outstanding) + self.assertLessEqual(peak, cap) + self.server.gate = other.gate = gate + future = self.drive_async(drive) + while not future.done(): + refs = reader.poll() + if refs: + # Acknowledge the actual completed outputs. Not-yet-returned + # writers remain in outstanding and continue occupying credit. + for ref in refs: + self.store.abort(ref.sample_id, reason='test-consumer-completed') + with lock: + outstanding -= sum(ref.estimated_bytes for ref in refs) + reader.mark_consumed(len(refs)) + time.sleep(.001) + self.assertEqual(future.result(timeout=3), 16) + self.assertLessEqual(peak, cap) + + def test_per_task_errors_retire_every_registered_attempt(self): + self.server.stub.error_sample_ids = {f'run0:p{i}' for i in range(4)} + workers, drive = self.build(n=4, max_prompt_attempts=1, + feature_store_max_resident_bytes=100000) + with self.assertRaisesRegex(RuntimeError, 'terminally failed prompt'): + drive() + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_fences) + self.assertFalse(self.store._capture_pending_begin) + self.assertFalse(self.backend._d) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_partial_lease_errors_cleanup_only_unreturned_identities(self): + self.server.stub.error_sample_ids = {'run0:p1', 'run0:p3'} + workers, drive = self.build(n=4, concurrency=2, lease=2, max_prompt_attempts=1, + feature_store_max_resident_bytes=100000) + with self.assertRaisesRegex(RuntimeError, 'terminally failed prompt'): + drive() + refs = self.channel.poll() + self.assertEqual({ref.sample_id for ref in refs}, {'run0:p0', 'run0:p2'}) + self.assertEqual(set(self.server.active), {'run0:p0', 'run0:p2'}) + self.assertTrue(self.backend._d) + self.assertEqual(set(self.store._capture_fences), {'run0:p0', 'run0:p2'}) + # This fixture negotiates legacy ownership, so successful fences stay + # live until consumer release; failed fences alone must be retired. + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def _stop_active_worker(self, method): + entered, release = threading.Event(), threading.Event() + calls = 0 + lock = threading.Lock() + def gate(action, body): + nonlocal calls + if action == 'generate': + with lock: + calls += 1 + if calls == 4: + entered.set() + self.assertTrue(release.wait(3)) + self.server.gate = gate + workers, drive = self.build(n=8) + future = self.drive_async(drive) + try: + self.assertTrue(entered.wait(2)) + getattr(workers[0], method)() + finally: + release.set() + with self.assertRaisesRegex(RuntimeError, 'stopped or draining with prompts remaining'): + future.result(timeout=3) + self.assertEqual(self.channel.published, 4) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + self.assertEqual(workers[0].controller.status()['prompts_pending'], 4) + + def test_stopped_worker_leaves_coordinator_without_spinning(self): + self._stop_active_worker('stop') + + def test_draining_worker_leaves_coordinator_without_spinning(self): + self._stop_active_worker('drain') + + def test_stopped_endpoint_releases_pending_plan_to_surviving_worker(self): + entered, release = threading.Event(), threading.Event() + other = BatchServer(self.backend) + def first_gate(action, body): + if action == 'generate': + entered.set() + self.assertTrue(release.wait(3)) + def other_gate(action, body): + if action == 'generate': + self.assertTrue(release.wait(3)) + self.server.gate, other.gate = first_gate, other_gate + workers, drive = self.build(n=12, concurrency=2, servers=[self.server, other]) + future = self.drive_async(drive) + try: + self.assertTrue(entered.wait(2)) + workers[0].stop() + finally: + release.set() + self.assertEqual(future.result(timeout=3), 12) + self.assertEqual(workers[0].health()['state'], 'stopped') + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_stop_after_registration_cleans_without_submitting_generate(self): + stop = threading.Event() + workers, drive = self.build(n=4, feature_store_max_resident_bytes=100000) + source = workers[0].feature_source + register = source.register_prepared_batches + def stopping_register(prepared): + register(prepared) + stop.set() + source.register_prepared_batches = stopping_register + self.assertEqual(drive(should_stop=stop.is_set), 0) + self.assertEqual([len(b['tasks']) for b in self.server.batches], [4]) + self.assertFalse(self.server.generates) + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + self.assertTrue(self.channel.is_closed()) + + def test_partial_preparation_is_loud_and_releases_every_lease(self): + workers, drive = self.build(n=4, max_worker_failures=1) + source = workers[0].feature_source + prepare = source.prepare_refs_batches + def short_prepare(groups, *, capture): + return prepare(groups, capture=capture)[:-1] + source.prepare_refs_batches = short_prepare + with self.assertRaisesRegex(RuntimeError, 'preparation returned 3 groups for 4 leases'): + drive() + self.assertFalse(self.server.batches) + self.assertFalse(self.server.generates) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_stop_releases_prepared_leases_waiting_for_credit(self): + per_token = sum(item['width'] * 2 for item in self.server.capability['schema']) + 16 + stop = threading.Event() + workers, drive = self.build(n=6, feature_store_max_resident_bytes=per_token*11) + future = self.drive_async(drive, should_stop=stop.is_set) + wait_for(lambda: self.channel.published >= 1) + stop.set() + self.assertGreaterEqual(future.result(timeout=3), 1) + self.assertFalse(self.server.active.keys() - {ref.sample_id for ref in self.channel.poll()}) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_capture_transfer.py b/tests/test_runtime/test_capture_transfer.py new file mode 100644 index 0000000000..a6d2f70876 --- /dev/null +++ b/tests/test_runtime/test_capture_transfer.py @@ -0,0 +1,256 @@ +"""Immutable published ownership, independent producer/consumer lifecycles.""" +import types +import unittest +from unittest import mock + +from specforge.inference.capture_protocol import CaptureProtocol, WriterFence, attempt_keys +from specforge.inference.adapters.server_capture import SGLangServerCaptureAdapter +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from tests.test_runtime.test_capture_fencing import ProtocolServer +from tests.test_runtime.test_server_capture import _FakeMooncakeStore, _task, _capture_schema, _eagle3_contract + + +class TransferServer(ProtocolServer): + def __init__(self, backend): + super().__init__(backend) + self.capability.update(begin_batch=True, published_ownership='attempt-v1') + self.calls = [] + + def __call__(self, url, json_body, timeout): + action = json_body.get('action', 'generate') + self.calls.append(action) + if action == 'begin_batch': + replies = [] + for body in json_body['tasks']: + reply = super().__call__(url, dict(body, action='begin'), timeout) + reply['object_key_layout'] = 'attempt-v1' + replies.append(reply) + return dict(attempts=replies) + if action == 'begin': + reply = super().__call__(url, json_body, timeout) + reply['object_key_layout'] = 'attempt-v1' + return reply + rows = super().__call__(url, json_body, timeout) + if action != 'generate': + return rows + for row in rows: + result = row['meta_info']['spec_capture'] + keys = attempt_keys(result, result['features']) + for name, key in keys.items(): + self.backend._d[key] = self.backend._d.pop(f"{result['store_id']}/{result['sample_id']}/g1/{name}") + result['features'][name]['key'] = key + result.update(object_key_layout='attempt-v1', publication_ownership='attempt-v1', physical_keys=keys) + entry = self.active.pop(result['sample_id']) + certificate = {k: result[k] for k in ('store_id', 'sample_id', 'gen', 'attempt_id', 'server_epoch', + 'reservation_token', 'object_key_layout', 'publication_ownership', 'physical_keys', 'writer_drained', 'state')} + entry.update(state='RETIRED', publication_certificate=certificate, + publication_ownership='attempt-v1', physical_keys=keys) + self.retired[entry['reservation_token']] = entry + return rows + + +class TransferTests(unittest.TestCase): + def setUp(self): + self.backend = _FakeMooncakeStore() + self.server = TransferServer(self.backend) + self.producer = MooncakeFeatureStore(store=self.backend, store_id='run0', max_hold_age_s=0) + self.adapter = SGLangServerCaptureAdapter('http://server', self.producer, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + + def produce(self, *indexes): + return self.adapter.produce_refs([_task(i, 5) for i in indexes], capture=_eagle3_contract()) + + def test_batched_begin_cached_capability_and_zero_consumer_rpc(self): + refs = self.produce(0, 1) + self.produce(2) + self.assertEqual(self.server.calls.count('capability'), 1) + self.assertEqual(self.server.calls.count('begin_batch'), 2) + self.assertNotIn('resolve', self.server.calls) + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + before = list(self.server.calls) + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('consumer control RPC')): + for ref in refs: + values, handle = consumer.get(ref) + self.assertEqual(values['hidden_state'].shape[1], 5) + consumer.release(handle) + consumer.drain_pending_removals(retry_interval_s=0) + self.assertEqual(before, self.server.calls) + + def test_multiple_readers_defer_local_remove(self): + ref = self.produce(0)[0] + _, first = self.producer.get(ref) + _, second = self.producer.get(ref) + self.producer.release(first) + self.assertTrue(self.backend._d) + self.assertEqual(len(self.producer._active_leases), 1) + self.producer.release(second) + self.assertFalse(self.backend._d) + + def test_producer_gc_handoff_and_stale_process_cannot_remove_replacement(self): + old = self.produce(0)[0] + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('consumer control RPC')): + _, handle = consumer.get(old) + self.producer.gc(now=10**12) + self.assertTrue(self.backend._d) + self.producer.relinquish_published(old) + self.assertFalse(self.producer._generation) + self.assertTrue(self.backend._d) + replacement_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + replacement = SGLangServerCaptureAdapter('http://server', replacement_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + new = replacement.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + self.assertNotEqual(old.feature_keys, new.feature_keys) + consumer.release(handle) # Only the old attempt's immutable keys. + self.assertEqual(set(self.backend._d), set(new.feature_keys.values())) + self.producer.abort(old.sample_id) + self.producer.gc(now=10**12) + self.assertEqual(set(self.backend._d), set(new.feature_keys.values())) + with self.assertRaises(KeyError): + self.producer.get(old) + + def test_lost_generate_response_resolves_certificate_then_cleans(self): + original = self.adapter.post_fn + def lose(*args, **kwargs): + original(*args, **kwargs) + raise TimeoutError('lost after successful publication') + self.adapter.post_fn = lose + with self.assertRaises(TimeoutError): + self.produce(0) + self.assertTrue(self.backend._d) + self.producer.discard_external_attempts() + self.producer.drain_pending_removals(retry_interval_s=0) + self.assertFalse(self.backend._d) + self.assertFalse(self.producer._capture_fences) + self.assertIn('resolve', self.server.calls) + self.assertNotIn('cleanup_acquire', self.server.calls) + + def test_strict_state_and_exact_keys_fail_closed(self): + ref = self.produce(0)[0] + fence = WriterFence(dict(ref.metadata['capture_fence']), tuple(ref.feature_keys)) + for bad in (dict(fence.wire, state='DRAINING'), dict(fence.wire, writer_drained=False), + dict(fence.wire, physical_keys={})): + with self.assertRaises(RuntimeError): + fence.accept_transfer(bad) + self.assertFalse(fence.transferred) + + def test_lost_begin_batch_reply_recovers_per_task_intents(self): + original = self.adapter._capture_protocol.post + once = [True] + def lose(url, json_body, timeout): + result = original(url, json_body, timeout) + if json_body.get('action') == 'begin_batch' and once[0]: + once[0] = False + raise TimeoutError('lost batched reservations') + return result + self.adapter._capture_protocol.post = lose + with self.assertRaises(TimeoutError): + self.produce(0, 1) + self.assertEqual(len(self.producer._capture_pending_begin), 2) + self.producer.discard_external_attempts() + self.assertFalse(self.server.active) + self.assertFalse(self.producer._capture_pending_begin) + + +class CapabilityTests(unittest.TestCase): + def test_absent_legacy_capability_is_cached(self): + calls = [] + def post(*args, **kwargs): + calls.append(1) + error = RuntimeError('404') + error.response = types.SimpleNamespace(status_code=404) + raise error + protocol = CaptureProtocol('http://server', post, 1) + self.assertIsNone(protocol.capability()) + self.assertIsNone(protocol.capability()) + self.assertEqual(len(calls), 1) + + def test_409_refreshes_capability_without_replaying_mutation(self): + calls = [] + def post(url, json_body, timeout): + calls.append(json_body['action']) + if json_body['action'] == 'capability': + return dict(protocol_version=1, server_epoch=str(len(calls)), capture_namespace='n', + capture_signature='s', schema=[], cache_enabled=False) + error = RuntimeError('409') + error.response = types.SimpleNamespace(status_code=409) + raise error + protocol = CaptureProtocol('http://server', post, 1) + protocol.capability() + with self.assertRaises(RuntimeError): + protocol.call(dict(action='begin')) + self.assertEqual(calls, ['capability', 'begin', 'capability']) + + +if __name__ == '__main__': + unittest.main() + +class PublicationBoundaryTests(TransferTests): + def test_visible_fsync_failure_relinquishes_prefix_and_aborts_suffix(self): + import tempfile + from specforge.launch import _publish_refs_with_cleanup + from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel + refs = self.produce(0, 1) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(directory + '/refs.jsonl') + with mock.patch('specforge.runtime.data_plane.streaming_ref_channel.os.fsync', side_effect=OSError('fsync failed')): + with self.assertRaises(OSError): + _publish_refs_with_cleanup(channel=channel, feature_store=self.producer, refs=refs) + self.assertEqual(channel.published, 1) + self.assertEqual([ref.sample_id for ref in channel.poll()], [refs[0].sample_id]) + self.assertEqual(set(self.backend._d), set(refs[0].feature_keys.values())) + self.assertNotIn(refs[0].sample_id, self.producer._capture_fences) + self.assertNotIn(refs[1].sample_id, self.producer._capture_fences) + + def test_same_consumer_accepts_new_attempt_and_rejects_old_ref(self): + old = self.produce(0)[0] + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('consumer control RPC')): + _, old_handle = consumer.get(old) + consumer.release(old_handle) + other_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + adapter = SGLangServerCaptureAdapter('http://server', other_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + new = adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + _, new_handle = consumer.get(new) + with self.assertRaises(KeyError): + consumer.get(old) + consumer.release(old_handle) + self.assertTrue(self.backend._d) + consumer.release(new_handle) + self.assertFalse(self.backend._d) + + def test_lost_generate_old_cleanup_cannot_touch_started_new_attempt(self): + original = self.adapter.post_fn + def lose(*args, **kwargs): + original(*args, **kwargs) + raise TimeoutError('lost old publication') + self.adapter.post_fn = lose + with self.assertRaises(TimeoutError): + self.produce(0) + other_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + adapter = SGLangServerCaptureAdapter('http://server', other_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + new = adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + self.producer.discard_external_attempts() + self.assertEqual(set(self.backend._d), set(new.feature_keys.values())) + +class DelayedAbortTests(TransferTests): + def test_ref_specific_old_abort_does_not_remove_same_store_replacement(self): + from specforge.runtime.data_plane.feature_store import abort_feature_ref + old = self.produce(0)[0] + self.producer.relinquish_published(old) + new = self.produce(0)[0] + before = set(self.backend._d) + abort_feature_ref(self.producer, old, reason='late old publication callback') + self.assertEqual(set(self.backend._d), before) + self.assertTrue(set(new.feature_keys.values()).issubset(self.backend._d)) + + def test_legacy_ref_cannot_abort_same_id_new_capture(self): + from dataclasses import replace + from specforge.runtime.data_plane.feature_store import abort_feature_ref + new = self.produce(0)[0] + legacy = replace(new, metadata={k: v for k, v in new.metadata.items() if k != 'capture_fence'}) + before = set(self.backend._d) + abort_feature_ref(self.producer, legacy, reason='late legacy callback') + self.assertEqual(set(self.backend._d), before) diff --git a/tests/test_runtime/test_native_hs_pipeline.py b/tests/test_runtime/test_native_hs_pipeline.py new file mode 100644 index 0000000000..9e68c0fc0b --- /dev/null +++ b/tests/test_runtime/test_native_hs_pipeline.py @@ -0,0 +1,116 @@ +"""Native opt-in and real coordinator/store CPU ownership contracts. + +The transport is an in-memory fixture; these tests do not certify CUDA, actual +Mooncake DMA, scheduler TP agreement or numerical feature equivalence. +""" +import os +import tempfile +import unittest +from unittest import mock + +from specforge.config.schema import DisaggregatedDeploymentConfig +from specforge.inference.adapters.server_capture import SGLangServerCaptureAdapter +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from tests.test_runtime.test_capture_transfer import TransferServer +from tests.test_runtime.test_disagg_multiserver import _adapter, _build, _prompts +from tests.test_runtime.test_server_capture import ( + _FakeMooncakeStore, _capture_schema, _eagle3_contract, _task, +) + + +class NativeHSConfiguration(unittest.TestCase): + def test_reuse_requires_protocol_and_mooncake(self): + for fields in (dict(hs_prefix_cache=True), dict(hs_capture=True, backend="shared_dir", store_root="/tmp/test"), dict(server_capture="patch")): + with self.subTest(fields=fields), self.assertRaises(ValueError): + DisaggregatedDeploymentConfig(**{ "control_dir": "/tmp/test", "backend": "mooncake", **fields}) + cfg = DisaggregatedDeploymentConfig(control_dir="/tmp/test", backend="mooncake", hs_capture=True) + self.assertFalse(cfg.hs_prefix_cache) + + def test_opt_in_rejects_missing_or_different_capability_before_begin(self): + for capability in (None, dict(published_ownership="attempt-v1", cache_enabled=False)): + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + adapter = SGLangServerCaptureAdapter("http://server", store, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), hs_capture=True, hs_prefix_cache=True) + with mock.patch.object(adapter._capture_protocol, "capability", return_value=capability), self.assertRaises(RuntimeError): + adapter.prepare_refs_batches([[_task(0, 5)]], capture=_eagle3_contract()) + self.assertFalse(store._capture_pending_begin) + self.assertFalse(backend._d) + + +class CapacityBackpressure(unittest.TestCase): + def test_atomic_refusal_retries_original_lease_without_prompt_failure(self): + for concurrency in (1, 3): + with self.subTest(concurrency=concurrency), tempfile.TemporaryDirectory() as root: + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + attempts = [] + refusal_count = 4 + + def post(url, json_body, timeout): + if json_body.get("action") == "begin_batch": + attempts.append(tuple(item["attempt_id"] for item in json_body["tasks"])) + if len(attempts) <= refusal_count: + return dict(error="full", error_code="capture_capacity_declined", reservation_granted=False) + return server(url, json_body, timeout) + + adapter = _adapter(store, post) + adapter.hs_capture = adapter.hs_prefix_cache = True + from specforge.inference.capture_protocol import CaptureProtocol + adapter._capture_protocol = CaptureProtocol(adapter.base_url, post, 1) + channel = StreamingRefChannel(os.path.join(root, "refs")) + workers, drive = _build([adapter], _prompts(3), store, channel, lease=1, producer_concurrency=concurrency, max_worker_failures=1, max_prompt_attempts=1, feature_store_max_resident_bytes=1 << 20, backpressure_poll_s=.001, peer_wait_timeout_s=2) + self.assertEqual(drive(), 3) + self.assertTrue(all(item == attempts[0] for item in attempts[:refusal_count + 1])) + self.assertEqual(workers[0].controller.status()["prompts_failed"], 0) + self.assertEqual(workers[0].health()["in_flight"], 0) + self.assertFalse(store._capture_pending_begin) + self.assertEqual(channel.published, 3) + + +class FencedReceiveDrain(unittest.TestCase): + def make_ref(self): + backend = _FakeMooncakeStore() + producer = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + adapter = SGLangServerCaptureAdapter("http://server", producer, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), post_fn=server, control_fn=server, hs_capture=True, hs_prefix_cache=True) + ref = adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + return backend, ref + + def test_failed_fenced_receive_keeps_buffer_registration_and_read_authority(self): + for failure in (-800, RuntimeError("late native read")): + with self.subTest(failure=failure): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0") + original_keys = set(backend._d) + effect = failure if isinstance(failure, Exception) else None + with mock.patch.object(backend, "get_into", side_effect=effect, return_value=failure) as read, mock.patch.object(backend, "unregister_buffer", wraps=backend.unregister_buffer) as unregister: + with self.assertRaises((KeyError, RuntimeError)): + consumer.get(ref) + self.assertEqual(read.call_count, 1) + self.assertEqual(unregister.call_count, 0) + self.assertEqual(consumer.health()["capture_failed_reads"], 1) + self.assertEqual(len(consumer._quarantined_buffers), 1) + self.assertEqual(len(consumer._capture_quarantined_registrations), 1) + with self.assertRaisesRegex(RuntimeError, "read ownership|readers|not drained"): + consumer.drain_pending_removals(retry_interval_s=0) + consumer.close() # Injected transport is still owned elsewhere. + self.assertEqual(consumer.health()["capture_failed_reads"], 1) + self.assertEqual(set(backend._d), original_keys) + + def test_owned_transport_close_is_required_before_releasing_failed_read(self): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0") + with mock.patch.object(backend, "get_into", return_value=-800), self.assertRaises(KeyError): + consumer.get(ref) + consumer._owns_store = True + with mock.patch.object(backend, "close", create=True, side_effect=RuntimeError("not stopped")): + with self.assertRaisesRegex(RuntimeError, "not stopped"): + consumer.close() + self.assertEqual(consumer.health()["capture_failed_reads"], 1) + with mock.patch.object(backend, "close", create=True, return_value=0): + consumer.close() + self.assertFalse(consumer._capture_read_intents) + self.assertFalse(consumer._quarantined_buffers) + self.assertFalse(consumer._capture_quarantined_registrations) diff --git a/tests/test_runtime/test_online_terminal_cleanup.py b/tests/test_runtime/test_online_terminal_cleanup.py new file mode 100644 index 0000000000..3573127cd6 --- /dev/null +++ b/tests/test_runtime/test_online_terminal_cleanup.py @@ -0,0 +1,226 @@ +"""Producer terminal scan reclaims handed-off refs only after proven success.""" +import ctypes +import os +import tempfile +import unittest +from pathlib import Path +from dataclasses import replace +from unittest import mock + +import torch + +from specforge.algorithms.builtin import builtin_algorithm_registry +from specforge.config import Config +from specforge.inference.capture_protocol import attempt_keys +from specforge.runtime.contracts import FeatureSpec, SampleRef +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from specforge.training.disaggregated import _build_online, _cleanup_online_published_refs +from tests.test_runtime.test_server_capture import _FakeMooncakeStore + + +class Backend(_FakeMooncakeStore): + def __init__(self): + super().__init__() + self.close_calls = 0 + self.remove_keys = [] + self.fail_remove = False + + def remove(self, key, **kwargs): + self.remove_keys.append(key) + return -1 if self.fail_remove else super().remove(key) + + def close(self): + self.close_calls += 1 + + +class TerminalCleanupTests(unittest.TestCase): + def setUp(self): + self.backend = Backend() + self.store = MooncakeFeatureStore(store_id='store', store=self.backend) + self.names = ('hidden', 'target', 'input_ids', 'loss_mask') + + def ref(self, sid, token): + wire = dict(protocol_version=1, store_id='store', sample_id=sid, gen=1, + server_epoch='epoch', attempt_id=token + '-attempt', reservation_token=token, + object_key_layout='attempt-v1', publication_ownership='attempt-v1', + writer_drained=True, state='PUBLISHED', control_url='http://forbidden/control') + wire['physical_keys'] = attempt_keys(wire, self.names) + data = torch.ones(1, 2, dtype=torch.float32) + raw = ctypes.string_at(data.data_ptr(), data.numel() * data.element_size()) + self.backend._d.update({key: raw for key in wire['physical_keys'].values()}) + return SampleRef(sample_id=sid, run_id='run', source_task_id=sid, + feature_store_uri='mooncake://store/' + sid, feature_keys=wire['physical_keys'], + feature_specs={name: FeatureSpec(name=name, shape=(1, 2), dtype='float32') for name in self.names}, + strategy='dflash', schema_version=1, target_model_version='target', tokenizer_version='tok', + num_tokens=2, estimated_bytes=32, metadata={'generation': 1, 'capture_fence': wire}) + + def handoff(self, ref): + self.store.adopt(ref) + self.store.relinquish_published(ref) + + def test_terminal_api_rejects_missing_success_or_closed_proof(self): + ref = self.ref('late', 'late-token') + self.handoff(ref) + for done, closed in ((False, False), (True, False), (False, True), (1, True)): + with self.subTest(done=done, closed=closed), self.assertRaisesRegex(RuntimeError, 'unresolved'): + self.store.cleanup_terminal_refs([ref], consumer_done=done, producer_closed=closed) + self.assertEqual(len(self.backend._d), 4) + self.assertIsNone(self.store._terminal_cleanup_authority) + self.assertEqual(self.backend.close_calls, 0) + + def test_terminal_scan_preserves_same_id_descendant_not_in_manifest(self): + old = self.ref('same', 'old') + self.handoff(old) + new = self.ref('same', 'new') + self.store.adopt(new) + new_keys = set(new.feature_keys.values()) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(str(Path(directory) / 'refs')) + channel.publish(old) + channel.mark_consumer_done() + channel.close() + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('terminal control RPC')): + _cleanup_online_published_refs(self.store, channel) + _cleanup_online_published_refs(self.store, channel) # Repeated scan is idempotent. + self.assertEqual(set(self.backend._d), new_keys) + self.assertEqual(self.store._capture_fences['same'].token, 'new') + self.assertEqual(self.backend.close_calls, 0) + + def test_failure_sentinel_wins_over_done_and_preserves_handoff_refs(self): + ref = self.ref('late', 'late') + self.handoff(ref) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(str(Path(directory) / 'refs')) + channel.publish(ref) + channel.mark_consumer_done() + channel.mark_consumer_failed('rank may still own a reader') + channel.close() + self.assertFalse(channel.consumer_done()) + self.assertTrue(channel.consumer_stopped()) + with self.assertRaisesRegex(RuntimeError, 'unresolved'): + _cleanup_online_published_refs(self.store, channel) + self.assertEqual(len(self.backend._d), 4) + self.assertEqual(self.backend.remove_keys, []) + + def test_terminal_identity_mismatch_never_falls_back_to_legacy_keys(self): + ref = self.ref('sample', 'token') + self.handoff(ref) + for bad in (replace(ref, sample_id='other'), + replace(ref, metadata={**ref.metadata, 'generation': 2})): + with self.subTest(sid=bad.sample_id, generation=bad.metadata['generation']), self.assertRaisesRegex(RuntimeError, 'identity mismatch'): + self.store.cleanup_terminal_refs([bad], consumer_done=True, producer_closed=True) + self.assertEqual(self.backend.remove_keys, []) + self.assertEqual(len(self.backend._d), 4) + + def test_terminal_uri_and_repeated_bad_proofs_are_validated_before_skip(self): + ref = self.ref('sample', 'token') + self.handoff(ref) + bad_uri = replace(ref, feature_store_uri='mooncake://different-store/sample') + with self.assertRaisesRegex(RuntimeError, 'URI identity mismatch'): + self.store.cleanup_terminal_refs([bad_uri], consumer_done=True, producer_closed=True) + self.assertEqual(self.backend.remove_keys, []) + self.assertIsNone(self.store._terminal_cleanup_authority) + self.store.cleanup_terminal_refs([ref], consumer_done=True, producer_closed=True) + removed = list(self.backend.remove_keys) + bad_generation = replace(ref, metadata={**ref.metadata, 'generation': 2}) + bad_state = replace(ref, metadata={**ref.metadata, 'capture_fence': { + **ref.metadata['capture_fence'], 'state': 'DRAINING'}}) + for bad in (bad_uri, bad_generation, bad_state): + with self.assertRaisesRegex(RuntimeError, 'unresolved'): + self.store.cleanup_terminal_refs([bad], consumer_done=True, producer_closed=True) + self.assertEqual(self.backend.remove_keys, removed) + + def test_missing_producer_eof_refuses_actual_terminal_scan(self): + ref = self.ref('late', 'late') + self.handoff(ref) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(str(Path(directory) / 'refs')) + channel.publish(ref) + channel.mark_consumer_done() + with self.assertRaisesRegex(RuntimeError, 'closed producer'): + _cleanup_online_published_refs(self.store, channel) + self.assertEqual(self.backend.remove_keys, []) + self.assertEqual(len(self.backend._d), 4) + + def test_failed_delete_keeps_cleanup_authority_and_retries_exact_keys(self): + ref = self.ref('late', 'late') + self.handoff(ref) + self.backend.fail_remove = True + drain_samples = MooncakeFeatureStore.drain_sample_removals + drain_pending = MooncakeFeatureStore.drain_pending_removals + def bounded_samples(store, ids): + return drain_samples(store, ids, max_attempts=1, retry_interval_s=0) + def bounded_pending(store): + return drain_pending(store, max_attempts=1, retry_interval_s=0) + with mock.patch.object(MooncakeFeatureStore, 'drain_sample_removals', bounded_samples), mock.patch.object(MooncakeFeatureStore, 'drain_pending_removals', bounded_pending): + with self.assertRaisesRegex(RuntimeError, 'unresolved'): + self.store.cleanup_terminal_refs([ref], consumer_done=True, producer_closed=True) + authority = self.store._terminal_cleanup_authority + self.assertIn(ref.sample_id, authority._release_pending) + self.assertEqual(len(self.backend._d), 4) + self.backend.fail_remove = False + with self.assertRaises(RuntimeError): + self.store.cleanup_terminal_refs([ref], consumer_done=False, producer_closed=True) + self.assertEqual(len(self.backend._d), 4) + self.store.cleanup_terminal_refs([ref], consumer_done=True, producer_closed=True) + self.assertIs(self.store._terminal_cleanup_authority, authority) + self.assertFalse(authority._release_pending) + self.assertFalse(self.backend._d) + self.assertEqual(self.backend.close_calls, 0) + + def test_real_produce_terminal_scan_removes_33_late_refs_and_accepts_already_acked(self): + refs = [self.ref(str(index), 'token-' + str(index)) for index in range(34)] + for ref in refs: + self.handoff(ref) + consumer = MooncakeFeatureStore(store_id='store', store=self.backend) + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('normal/terminal control RPC')): + _, handle = consumer.get(refs[0]) + consumer.release(handle) + self.assertEqual(len(self.backend._d), 33 * 4) + with tempfile.TemporaryDirectory() as directory: + channel_path = str(Path(directory) / 'refs') + cfg = Config.model_validate({ + 'model': {'target_model_path': 'target', 'draft_model_config': 'draft'}, + 'data': {'prompts_path': 'unused'}, + 'training': {'strategy': 'dflash', 'role': 'producer', 'max_steps': 30, 'total_steps': 1197}, + 'deployment': {'mode': 'disaggregated', 'disaggregated': { + 'control_dir': directory, 'backend': 'mooncake', 'server_urls': ['http://capture']}}}) + def drive(*, should_stop): + self.assertFalse(should_stop()) + channel = StreamingRefChannel(channel_path) + channel.publish(refs[0]) + channel.mark_consumer_done() + # Match the observed race: completed captures are published + # after consumer_done, then workers finish and EOF is sealed. + for ref in refs[1:]: + channel.publish(ref) + channel.close() + return len(refs) + with ( + mock.patch.dict(os.environ, {'DISAGG_REF_CHANNEL': channel_path}), + mock.patch('specforge.training.disaggregated._mooncake_store', return_value=self.store), + mock.patch('specforge.training.assembly._load_input_tools'), + mock.patch('specforge.training.model_loading.resolve_draft_config'), + mock.patch('specforge.training.disaggregated._producer_capture_metadata', return_value=([1], 2, 4, 4)), + mock.patch('specforge.inference.adapters.server_capture.SGLangServerCaptureAdapter'), + mock.patch('specforge.launch.build_disagg_online_producer', return_value=([], drive)), + ): + run = _build_online(cfg, algorithm=builtin_algorithm_registry().resolve('dflash'), + build_model_bundle=mock.Mock(), + prepare_prompts=lambda *args, **kwargs: [{'task_id': 'p', 'payload': {'input_ids': [1, 2]}}], + optimizer_factory=mock.Mock(), logger=None) + self.assertEqual(run.run(), 34) + self.assertFalse(self.backend._d) + self.assertEqual(self.backend.close_calls, 0) + authority = self.store._terminal_cleanup_authority + self.assertIsNotNone(authority) + self.assertFalse(authority._owns_store) + self.assertFalse(authority._release_pending) + self.assertFalse(authority._capture_fences) + with self.assertRaises(KeyError): + self.store.adopt(refs[-1]) # Normal handoff tombstones were not reopened. + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_server_capture_gate.py b/tests/test_runtime/test_server_capture_gate.py index 495e1f093d..8065682dc7 100644 --- a/tests/test_runtime/test_server_capture_gate.py +++ b/tests/test_runtime/test_server_capture_gate.py @@ -178,11 +178,11 @@ def setUpClass(cls): env = dict( os.environ, - FLASHINFER_DISABLE_VERSION_CHECK="1", MOONCAKE_LOCAL_HOSTNAME=os.environ.get( "MOONCAKE_LOCAL_HOSTNAME", "127.0.0.1" ), ) + env.pop("FLASHINFER_DISABLE_VERSION_CHECK", None) if SERVER_CAPTURE == "plugin": env["SPECFORGE_SPEC_CAPTURE"] = "1" server_python = os.environ.get("SPECFORGE_CAPTURE_GATE_SERVER_PYTHON") From 8fddf464b015f3ede1276731236fa021b2083094 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 02:04:40 +0800 Subject: [PATCH 05/17] feat(capture): integrate fenced HS ownership with native outputs Implements frozen interface v1: plugin-owned scheduler callbacks, atomic attempt registry/control routing, exact model/schema namespaces, compact suffix snapshots, sole bounded controller D2H, complete N-row assembly, batched sink publication and native deferred completion. Sink uses immutable attempt keys, drains argument references before cache adoption, and retains sources/registrations/charges for unknown native write drain. Adds focused CPU adversarial contracts and explicit supported-mode metadata. Sources: frozen 0b3e00aa attempt/control ownership semantics; SF224b36bd native observer/aux-output/completion/sink remains the integration path. AST and whitespace checks only; independent CPU/GPU tests pending. --- plugins/sglang-spec-capture/README.md | 61 ++ .../specforge_sglang_capture/__init__.py | 12 + .../specforge_sglang_capture/capture.py | 60 +- .../specforge_sglang_capture/hs/attempts.py | 137 +++ .../specforge_sglang_capture/hs/config.py | 41 + .../specforge_sglang_capture/hs/http.py | 30 + .../specforge_sglang_capture/hs/runtime.py | 865 ++++++++++++++++++ .../specforge_sglang_capture/sink.py | 132 ++- tests/test_runtime/test_hs_plugin.py | 304 ++++++ 9 files changed, 1597 insertions(+), 45 deletions(-) create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py create mode 100644 plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py create mode 100644 tests/test_runtime/test_hs_plugin.py diff --git a/plugins/sglang-spec-capture/README.md b/plugins/sglang-spec-capture/README.md index 449702bc38..d52c7be29c 100644 --- a/plugins/sglang-spec-capture/README.md +++ b/plugins/sglang-spec-capture/README.md @@ -180,3 +180,64 @@ with RDMA NICs. Without it, Mooncake installs its RDMA transport instead. | `SGLANG_SPEC_CAPTURE_ARENA_BYTES` | Arena size in bytes. Required with `MOONCAKE_PROTOCOL=nvlink`. | | `SGLANG_SPEC_CAPTURE_CONTROL_PORT` | Port of the NVLink free endpoint (default: ephemeral). | | `MOONCAKE_*` | Mooncake client settings. | + +## Opt-in HS capture protocol v1 + +This migration additionally needs SGLang's native +`managers.prefix_admission.CAPTURE_ADMISSION_PROTOCOL == 1` companion extension. +It does not apply the removed legacy capture patch. Enable +`SPECFORGE_HS_CAPTURE=1` to negotiate server-issued attempt reservations at +`POST /spec_capture/control`; set `SPECFORGE_HS_CACHE=1` to reuse hidden-state +prefixes. `HS_CAPTURE=1, HS_CACHE=0` keeps the same complete-output, byte-budget +and immutable-attempt protocol while forcing a zero prefix. The ordinary +non-negotiated capture path remains available when both are off. + +| Variable | Default | +|---|---:| +| `SPECFORGE_HS_CHUNK_TOKENS` | 256 | +| `SPECFORGE_HS_CACHE_BYTES` | 1073741824 | +| `SPECFORGE_HS_PINNED_BYTES` | 268435456 | +| `SPECFORGE_HS_OUTPUT_BYTES` | 536870912 | +| `SPECFORGE_HS_DEVICE_BYTES` | 268435456 | +| `SPECFORGE_HS_MAX_WORKERS` | 4 | +| `SPECFORGE_HS_ATTEMPT_CAPACITY` | 4096 | + +All limits must be positive. Pinned staging uses one retained arena, conservatively +rounded down to a power of two, with aligned independent reservation slots. +Cache adoption charges complete underlying storage, including unmatchable tails. +Output, pinned and device limits are separate. Hard oversize fails explicitly; +temporary admission pressure permits only the native transaction's single +fully-rolled-back reclaim/retry. Atomic registry-capacity rejection returns +`error_code=capture_capacity_declined, reservation_granted=false` before granting +any attempt in that begin operation. + +HS capture supports complete causal text, nonchunked prefill, PP1/DP1, no +offload/session/LoRA/weight updates, and host publication with Mooncake hard pin. +Set `SGLANG_SPEC_CAPTURE_GPU_PUT=0` when using RDMA host staging. Reuse additionally +requires deterministic batch-invariant inference, enabled radix, and an exact +selected recurrent checkpoint backend. Unsupported modes fail startup rather +than changing capture semantics. Capability metadata advertises these limits. + +The content namespace binds model/config/load epoch, ordered raw auxiliary +layers, post-norm last hidden state, dtype/layout, causal position/attention, +TP/KV/checkpoint/backend and deterministic semantics. Tokens form complete parent +hash chains; sample/attempt IDs and loss masks do not enter content keys. +Passthrough data remains part of the registered request digest. + +Each external attempt has immutable keys +`store/sample/g1/attempt/server_epoch/reservation_token/feature`. The native +observer retains independently owned suffix snapshots; its auxiliary output +carries controller jobs and performs no second D2H. Assembly reconstructs every +row `[0,N)` from the leased prefix and the current `[L,N)` suffix. Scheduler safe +points batch ready samples into the native sink, then resolve native deferred +responses. A PUBLISHED certificate is issued only after exact key/schema/digest +validation and source-reader drain. Sink completion permits cache adoption; it +does not retire storage retained by cache leases. Unknown DMA/native transfer +drain quarantines sources, registrations and charges until an external drain +proof exists. Timeouts do not authorize reuse. + +`tests.test_runtime.test_hs_runtime` covers the migrated CPU core; +`tests.test_runtime.test_hs_plugin` covers native-output wiring with a CPU buffer +store; `test_hs_controller_parallel_cuda` requires actual CUDA/pinned copies. +CPU fixtures do not certify numerical model, real Mooncake, scheduler TP or +training equivalence. diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py b/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py index 19927b8c99..070ee2fa6c 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py @@ -44,6 +44,14 @@ def register() -> None: HookRegistry.register( f"{_SCHEDULER}.get_internal_state", _capture_internal_state, HookType.AFTER ) + from specforge_sglang_capture.hs.config import HSConfig + if HSConfig.from_env().capture: + from specforge_sglang_capture.hs.http import install_control_route + HookRegistry.register( + "sglang.srt.entrypoints.http_server.set_global_state", + install_control_route, + HookType.AFTER, + ) def _capture_streamer_class(streamer_class, scheduler, *args, **kwargs): @@ -69,6 +77,10 @@ def _capture_internal_state(result, scheduler, *args, **kwargs): result.internal_state["spec_capture_backend"] = "plugin" result.internal_state["spec_capture_method"] = runtime.capture_method result.internal_state["spec_capture_aux_layer_ids"] = runtime.sink.aux_layer_ids + hs = getattr(runtime, "hs", None) + if hs is not None: + result.internal_state["spec_capture_hs_protocol"] = 1 + result.internal_state["spec_capture_hs_cache"] = hs.enabled if runtime.is_writer: control = runtime.sink.control_endpoint() if control is not None: diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py b/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py index e6be261553..4e30e7d337 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py @@ -117,12 +117,19 @@ def collect_capture_items( if spec is not None: error = spec.get("_parse_error") prompt_len = len(req.origin_input_ids) - if error is None and prefix_len: + plan = getattr(req, "_specforge_hs_plan", None) + if error is None and plan is not None: + if (prefix_len != plan.L or length != plan.N - plan.L + or prompt_len != plan.N or plan.execution.request_id != req.rid): + error = "the native capture span differs from its committed HS plan" + elif error is None and "protocol_version" in spec: + error = "HS capture has no committed native admission plan" + elif error is None and prefix_len: error = ( f"{prefix_len} prompt tokens were served from the prefix " "cache; spec capture needs a full prefill" ) - elif error is None and length != prompt_len: + elif error is None and plan is None and length != prompt_len: error = ( f"the prefill covered {length} of {prompt_len} prompt tokens; " "launch the server with --chunked-prefill-size -1" @@ -185,9 +192,25 @@ def after_forward(self, batch, forward_batch, logits_output, *, can_run_graph): ), ) checked.append(item) - return CaptureDeviceOutput.from_rows( - self.runtime, checked, aux, last_hidden, copy=can_run_graph - ) + jobs, native_items = [], [] + reqs = {req.rid: req for req in batch.reqs} + for item in checked: + if self.runtime.hs is None or "protocol_version" not in item.spec: + native_items.append(item) + continue + req = reqs[item.rid] + if item.error is None: + try: + jobs.append(self.runtime.hs.snapshot(req, item, aux, last_hidden)) + continue + except Exception as exc: + item = dataclasses.replace(item, error=str(exc)) + self.runtime.hs.capture_error(req, item.error) + native_items.append(item) + output = CaptureDeviceOutput.from_rows( + self.runtime, native_items, aux, last_hidden, copy=can_run_graph) + output.hs_jobs = jobs + return output def _select_rows( @@ -235,6 +258,7 @@ def __init__(self, runtime, items, aux, last_hidden): self.items = items self.aux = aux self.last_hidden = last_hidden + self.hs_jobs = [] @classmethod def from_rows(cls, runtime, items, aux, last_hidden, *, copy): @@ -242,30 +266,36 @@ def from_rows(cls, runtime, items, aux, last_hidden, *, copy): return cls(runtime, items, aux, last_hidden) def copy_to_host(self, copy_tensor): + if not self.items: + # HS controller owns the only D2H path and its bounded pinned arena. + return CaptureHostOutput(self.runtime, [], None, None, hs_jobs=self.hs_jobs) if self.runtime.gpu_put: # Device publication: the sink waits for everything enqueued so far. ready = torch.cuda.Event() ready.record() return CaptureHostOutput( - self.runtime, self.items, self.aux, self.last_hidden, ready_event=ready + self.runtime, self.items, self.aux, self.last_hidden, ready_event=ready, + hs_jobs=self.hs_jobs, ) return CaptureHostOutput( self.runtime, self.items, copy_tensor(self.aux) if self.aux is not None else None, copy_tensor(self.last_hidden) if self.last_hidden is not None else None, + hs_jobs=self.hs_jobs, ) class CaptureHostOutput: """HostAuxiliaryOutput: publish finished capture requests, then hold them.""" - def __init__(self, runtime, items, aux, last_hidden, *, ready_event=None): + def __init__(self, runtime, items, aux, last_hidden, *, ready_event=None, hs_jobs=()): self.runtime = runtime self.items = items self.aux = aux self.last_hidden = last_hidden self.ready_event = ready_event + self.hs_jobs = hs_jobs def _rows(self, tensor, item): if tensor is None: @@ -273,6 +303,9 @@ def _rows(self, tensor, item): return tensor[item.start : item.start + item.length] def consume(self, batch, commits) -> None: + if self.hs_jobs: + self.runtime.hs.consume(self.hs_jobs, batch) + self.hs_jobs = [] reqs = {req.rid: req for req in batch.reqs} samples: List[Sample] = [] held = [] @@ -433,6 +466,7 @@ def __init__( self.results: Dict[str, Dict[str, Any]] = {} self.pending = PendingCaptures(self, max_pending_batches) self.observer = CaptureObserver(self) + self.hs = None def install(scheduler) -> CaptureRuntime: @@ -485,6 +519,18 @@ def install(scheduler) -> CaptureRuntime: ) method = features.aux_hidden_state_capture runtime.capture_method = str(getattr(method, "name", method)).lower() + from specforge_sglang_capture.hs.config import HSConfig + hs_config = HSConfig.from_env() + if hs_config.capture: + from sglang.srt.managers.prefix_admission import CAPTURE_ADMISSION_PROTOCOL + from specforge_sglang_capture.hs.runtime import HSCaptureRuntime + + if CAPTURE_ADMISSION_PROTOCOL != 1: + raise RuntimeError("HS capture requires native admission protocol v1") + if getattr(scheduler, "prefix_admission_observer", None) is not None: + raise RuntimeError("another native prefix admission observer is already installed") + runtime.hs = HSCaptureRuntime(scheduler, runtime, hs_config) + scheduler.prefix_admission_observer = runtime.hs model_runner.forward_observer = runtime.observer scheduler.output_streamer.spec_capture_results = runtime.results scheduler._specforge_capture_runtime = runtime diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py new file mode 100644 index 0000000000..e760d4d7e6 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py @@ -0,0 +1,137 @@ +"""Scheduler-owned immutable attempt registry, sourced from frozen HS v1.""" +from __future__ import annotations + +import hashlib +import json +import uuid +from collections import deque +from dataclasses import dataclass, field +from typing import Any + + +class CaptureControlRejected(ValueError): + # Structured marker also survives injected in-process control transports. + capture_control_rejected = True + + +class AttemptCapacityDeclined(CaptureControlRejected): + """An atomic begin operation has not granted any reservations.""" + + +def scalar_error(exc): + """Detach completed worker errors from frames that may retain tensor storage.""" + message = str(exc) + exc.__traceback__ = exc.__cause__ = exc.__context__ = None + exc.args = (message,) + error = RuntimeError(message) + if getattr(exc, "capture_writer_drained", True) is False: + error.capture_writer_drained = False + return error + + +def digest_request(tokens, spec): + value = {"tokens": list(tokens), "features": spec.get("features", {}), + "passthrough": spec.get("passthrough", [])} + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +@dataclass +class Attempt: + identity: tuple + attempt_id: str + request_digest: str + token: str + sequence: int + state: str = "RESERVED" + revoked: bool = False + claimed: bool = False + request: Any = None + execution_epoch: int = 0 + drains: list = field(default_factory=list) + deliveries: int = 0 + last_plan: dict = field(default_factory=dict) + object_key_layout: str | None = None + transfer_certificate: dict = field(default_factory=dict) + cleanup_owner: str | None = None + cleanup_token: str | None = None + readers: dict = field(default_factory=dict) + released_readers: deque = field(default_factory=deque) + + +class AttemptRegistry: + """Bounded active reservations. Unknown tokens never authorize generation.""" + def __init__(self, epoch, capacity=4096): + if capacity < 1: + raise ValueError("attempt capacity must be positive") + self.epoch, self.capacity = epoch, capacity + self.active = {} + self.tokens = {} + self.retired = deque(maxlen=capacity) + self.sequence = 0 + + @staticmethod + def identity(body): + identity = (str(body["store_id"]), str(body["sample_id"]), int(body.get("gen", 1))) + if identity[2] != 1: + raise ValueError("capture protocol requires gen=1") + return identity + + def begin(self, body, token=None): + identity = self.identity(body) + old = self.active.get(identity) + attempt_id, digest = str(body["attempt_id"]), str(body["request_digest"]) + if not attempt_id or not digest: + raise ValueError("attempt id and request digest are required") + if old: + if old.attempt_id != attempt_id or old.request_digest != digest: + raise ValueError("prior sample attempt is not retired") + if old.revoked: + raise ValueError("attempt is revoked; resolve and clean it before retry") + return old + if len(self.active) >= self.capacity: + raise AttemptCapacityDeclined("attempt registry capacity exhausted") + self.sequence += 1 + entry = Attempt(identity, attempt_id, digest, token or uuid.uuid4().hex, self.sequence) + entry.object_key_layout = "attempt-v1" if body.get("publication_ownership") == "attempt-v1" else None + self.active[identity] = entry + self.tokens[entry.token] = entry + return entry + + def begin_batch(self, bodies, tokens): + # Validate the complete batch before changing the live registry. + if len(bodies) != len(tokens) or len({self.identity(body) for body in bodies}) != len(bodies): + raise ValueError("duplicate begin_batch identity or token count mismatch") + trial = AttemptRegistry(self.epoch, self.capacity) + trial.active, trial.tokens, trial.sequence = dict(self.active), dict(self.tokens), self.sequence + for body, token in zip(bodies, tokens): + trial.begin(body, token) + self.active, self.tokens, self.sequence = trial.active, trial.tokens, trial.sequence + return [self.active[self.identity(body)] for body in bodies] + + def lookup(self, body): + token = str(body["reservation_token"]) + entry = self.tokens.get(token) + if entry is None or entry.identity != self.identity(body) or entry.attempt_id != str(body["attempt_id"]): + raise ValueError("unknown, retired or mismatched reservation token") + return entry + + def claim(self, body, tokens): + entry = self.lookup(body) + if entry.revoked or entry.claimed: + raise ValueError("reservation revoked or generation already claimed") + if digest_request(tokens, body) != entry.request_digest: + raise ValueError("generate payload differs from reservation") + entry.claimed, entry.state = True, "QUEUED" + return entry + + def retire(self, entry): + if entry.state not in ("WRITER_DRAINED", "PUBLISHED") or entry.deliveries or entry.readers: + raise ValueError("writer is not drained") + if any(not handle.done() for handle in entry.drains): + raise ValueError("runtime readers are not drained") + entry.revoked, entry.state = True, "RETIRED" + self.tokens.pop(entry.token, None) + self.active.pop(entry.identity, None) + entry.request = None + entry.drains.clear() + self.retired.append(entry) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py new file mode 100644 index 0000000000..4710f52988 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py @@ -0,0 +1,41 @@ +"""Explicit native-plugin HS configuration, independent of legacy SG flags.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _flag(name: str) -> bool: + value = os.environ.get(name, "0") + if value not in ("0", "1"): + raise ValueError(f"{name} must be 0 or 1") + return value == "1" + + +@dataclass(frozen=True) +class HSConfig: + capture: bool = False + cache: bool = False + chunk_tokens: int = 256 + cache_bytes: int = 1 << 30 + pinned_bytes: int = 256 << 20 + output_bytes: int = 512 << 20 + device_bytes: int = 256 << 20 + max_workers: int = 4 + attempt_capacity: int = 4096 + + @classmethod + def from_env(cls) -> HSConfig: + values = {"capture": _flag("SPECFORGE_HS_CAPTURE"), + "cache": _flag("SPECFORGE_HS_CACHE")} + for name in ("chunk_tokens", "cache_bytes", "pinned_bytes", "output_bytes", + "device_bytes", "max_workers", "attempt_capacity"): + env = "SPECFORGE_HS_" + name.upper() + value = int(os.environ.get(env, str(getattr(cls, name)))) + if value <= 0: + raise ValueError(f"{env} must be positive") + values[name] = value + if values["cache"] and not values["capture"]: + raise ValueError("SPECFORGE_HS_CACHE=1 requires SPECFORGE_HS_CAPTURE=1") + return cls(**values) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py new file mode 100644 index 0000000000..3eed995c8f --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py @@ -0,0 +1,30 @@ +"""HTTP forwarding only; all HS state changes execute on TP schedulers.""" + +from __future__ import annotations + +from fastapi import Request +from fastapi.responses import JSONResponse + + +def install_control_route(result, *args, **kwargs): + from sglang.srt.entrypoints import http_server + + app = http_server.app + if any(getattr(route, "path", None) == "/spec_capture/control" for route in app.routes): + return result + + async def spec_capture_control(request: Request): + body = await request.json() + if not isinstance(body, dict): + return JSONResponse({"error": "capture control requires a JSON object"}, status_code=400) + manager = http_server.get_global_state().tokenizer_manager + try: + # plugin_control owns the request/reply communicator. It invokes + # runtime.control on every scheduler rank in the same phase order. + reply = await manager.plugin_control("specforge_hs", body) + except (ValueError, RuntimeError) as exc: + return JSONResponse({"error": str(exc)}, status_code=409) + return JSONResponse(reply) + + app.add_api_route("/spec_capture/control", spec_capture_control, methods=["POST"]) + return result diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py new file mode 100644 index 0000000000..349c546b72 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py @@ -0,0 +1,865 @@ +"""Native capture admission and publication, owned by the scheduler thread. + +SG owns KV/SSM admission and TP transaction receipts. This module owns only HS +leases, output allocations and immutable external-attempt publication. Assembly +workers and sink callbacks never read a Req or execute a collective. +""" + +from __future__ import annotations + +import copy +import dataclasses +import hashlib +import json +import logging +import os +import threading +import time +import uuid +from collections import deque +from concurrent.futures import Future +from typing import Any + +import torch + +from .attempts import ( + Attempt, AttemptCapacityDeclined, AttemptRegistry, CaptureControlRejected, + digest_request, scalar_error, +) +from .cache import PrefixFeatureCache +from .config import HSConfig +from .controller import CaptureController +from .protocol import ( + CaptureContractError, CapturedSpan, ExecutionIdentity, FeatureSchema, + FeatureSpec, OutputReservation, PrefixLease, +) + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class _Candidate: + execution: ExecutionIdentity + entry: Attempt + tokens: tuple[int, ...] + lease: PrefixLease | None = None + reservation: OutputReservation | None = None + L: int = 0 + + +@dataclasses.dataclass +class _Publication: + execution: ExecutionIdentity + entry: Attempt + spec: dict + plan: Any + materialize: Future + output: Future + consumed: bool = False + sample: Any = None + sink: Future | None = None + sink_index: int = 0 + sink_count: int = 0 + cleanup: Future | None = None + failed: BaseException | None = None + publish_started: float = 0.0 + + +class HSCompletion: + """Native ResponseCompletion for one immutable execution. + + Cancelling a response fences its execution. The runtime retains the job + until materialization and any shared sink reader really stop. + """ + + def __init__(self, runtime, job): + self.runtime, self.job = runtime, job + self._cancelled = self._resolved = False + + def done(self): + return self._cancelled or self.job.output.done() + + def result(self): + if self._cancelled or self._resolved: + return + self.runtime._assert_owner() + if not self.runtime._same_execution(self.job): + self._resolved = True + return + try: + result = self.job.output.result() + except Exception as exc: + result = {"sample_id": self.job.spec["sample_id"], "error": str(exc)} + self.runtime.capture.results[self.job.execution.request_id] = result + self._resolved = True + + def cancel(self): + if self._resolved: + return False + if not self._cancelled: + self.runtime._assert_owner() + self.runtime._cancel_job(self.job, "deferred-response-cancelled") + self._cancelled = True + return True + + +class HSCaptureRuntime: + control_namespace = "specforge_hs" + + def __init__(self, scheduler, capture, config: HSConfig, *, manifest=None, + schema=None, tp_size=None): + self.s, self.capture, self.config = scheduler, capture, config + self.capture.sink.require_hard_pin = True + self.writer, self.enabled = capture.is_writer, config.cache + self.owner_thread = threading.get_ident() + self.tp_size = tp_size + if self.tp_size is None: + from sglang.srt.runtime_context import get_parallel + self.tp_size = get_parallel().tp_size + self.epoch = self.broadcast(uuid.uuid4().hex if self.writer else None) + if manifest is None or schema is None: + manifest, schema = self._model_identity() + self.schema = schema + self.schema_wire = [dataclasses.asdict(feature) for feature in schema.features] + for feature in self.schema_wire: + feature["layer_ids"] = list(feature["layer_ids"]) + manifest = {**manifest, "load_epoch": self.epoch, "schema": self.schema_wire, + "chunk_tokens": config.chunk_tokens} + self.signature = hashlib.sha256(json.dumps( + manifest, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest() + self.namespace = "spec-feature-v1:" + self.signature + if len(set(self.gather(self.signature))) != 1: + raise CaptureContractError("TP capture model/schema signatures differ") + self.registry = AttemptRegistry(self.epoch, config.attempt_capacity) + self.cache = self.controller = None + if self.writer: + self.cache = PrefixFeatureCache(schema, config.chunk_tokens, + config.cache_bytes if self.enabled else 0) + self.controller = CaptureController( + self.cache, schema, config.pinned_bytes, config.output_bytes, + config.device_bytes, max_workers=config.max_workers) + self.candidates: dict[str, _Candidate] = {} + self.jobs: dict[ExecutionIdentity, _Publication] = {} + self.history = deque(maxlen=256) + self._transferred_tokens = [] + self._closed = False + self.fatal_capture_error = None + self.quarantined_snapshots = [] + self.quarantined_publications = [] + + def _assert_owner(self): + if threading.get_ident() != self.owner_thread: + raise RuntimeError("HS state may only be changed on the scheduler thread") + + def broadcast(self, value): + if self.tp_size == 1: + return value + if hasattr(self.s, "prefix_admission_broadcast"): + return self.s.prefix_admission_broadcast(value) + import torch.distributed as dist + values = [value] + dist.broadcast_object_list(values, src=self.s.tp_group.ranks[0], + group=self.s.tp_cpu_group) + return values[0] + + def gather(self, value): + if self.tp_size == 1: + return [value] + if hasattr(self.s, "prefix_admission_gather"): + return self.s.prefix_admission_gather(value) + import torch.distributed as dist + values = [None] * self.tp_size + dist.all_gather_object(values, value, group=self.s.tp_cpu_group) + return values + + def _model_identity(self): + from sglang.srt.arg_groups.model_override_base import resolved_view + from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule + from specforge_sglang_capture.sink import gpu_put_enabled + + args = resolved_view(self.s.server_args) + parallel = get_parallel() + schedule = get_schedule() + features = get_exec().features + forbidden = { + "chunked_prefill": schedule.chunked_prefill_size != -1, + "PP": parallel.pp_size != 1, + "DP": parallel.dp_size != 1 or bool(args.enable_dp_attention), + "CP": parallel.attn_cp_size != 1 or parallel.dcp_size != 1, + "HiCache": bool(args.enable_hierarchical_cache), + "LoRA": bool(args.enable_lora), + "sessions": bool(args.enable_session_radix_cache) + or bool(getattr(args, "enable_streaming_session", False)), + "offload": any(bool(getattr(args, name, False)) for name in ( + "hicache_storage_backend", "enable_lmcache", "enable_flexkv", + "disaggregation_decode_enable_offload_kvcache")), + "GPU_publication": gpu_put_enabled(), + "priority_preemption": bool(self.s.enable_priority_preemption), + "prefill_delayer": bool(self.s.prefill_delayer) + or bool(getattr(self.s, "min_free_slots_delayer", None)), + "DLLM": self.s.dllm_config is not None, + "speculative_decode": not self.s.spec_algorithm.is_none(), + "radix_disabled": self.enabled and bool(args.disable_radix_cache), + "memory_saver": bool(args.enable_memory_saver), + "deferred_weights": bool(getattr(args, "wait_for_weights_before_ready", False)), + } + if bad := [name for name, present in forbidden.items() if present]: + raise CaptureContractError("HS capture unsupported configuration: " + ", ".join(bad)) + if self.enabled: + if not args.enable_deterministic_inference: + raise CaptureContractError("HS reuse requires --enable-deterministic-inference") + incompatible = [name for name in ( + "enable_int8_mamba_checkpoint", "enable_mamba_cache_stochastic_rounding", + "enable_linear_replayssm", "enable_linear_replayssm_spec", + ) if getattr(args, name, False)] + if incompatible: + raise CaptureContractError("HS reuse requires exact recurrent checkpoints: " + + ", ".join(incompatible)) + from sglang.srt.managers.prefix_admission import validate_capture_checkpoint_support + checkpoint = validate_capture_checkpoint_support(self.s) + else: + checkpoint = {"reuse": False} + if self.config.chunk_tokens % self.s.page_size: + raise CaptureContractError("HS chunk tokens must be a multiple of the KV page size") + layers = tuple(features.return_aux_hidden_state_layer_ids or ()) + if not layers: + raise CaptureContractError("HS capture needs ordered auxiliary layer IDs") + text = self.s.model_config.hf_text_config + width = int(text.hidden_size) + dtype = str(self.s.model_config.dtype).removeprefix("torch.") + schema = FeatureSchema((FeatureSpec("aux", width * len(layers), dtype, layers), + FeatureSpec("last_hidden", width, dtype, ()))) + manifest = { + "model": os.path.realpath(args.model_path), "revision": args.revision, + "config": text.to_dict(), "dtype": dtype, + "quantization": args.quantization, "load_format": str(args.load_format), + "weights": "immutable-for-load-epoch", "tp_size": parallel.tp_size, + "deterministic_inference": bool(args.enable_deterministic_inference), + "kv_dtype": args.kv_cache_dtype, "bf16_gemm_backend": args.bf16_gemm_backend, + "attention_backend": args.attention_backend, + "prefill_attention_backend": args.prefill_attention_backend, + "linear_attention_backend": getattr(args, "linear_attn_backend", None), + "checkpoint_semantics": checkpoint, + "capture_method": str(features.aux_hidden_state_capture), + "aux_semantics": "raw_ordered_capture", "last_hidden_semantics": "post_norm", + "layout": "contiguous_token_width", "positions": "standard_text_causal", + } + return manifest, schema + + def capability(self): + return {"protocol_version": 1, "server_epoch": self.epoch, + "begin_batch": True, "published_ownership": "attempt-v1", + "capture_namespace": self.namespace, "capture_signature": self.signature, + "schema": self.schema_wire, "cache_enabled": self.enabled, + "transport": "host", "no_oversize_fallback": True, + "support": {"nonchunked": True, "host_publication": True, + "causal_text_only": True, "pp_size": 1, "dp_size": 1, + "offload": False, "sessions": False, "lora": False, + "weight_updates": False, "complete_schema": True}, + "limits": {name: getattr(self.config, name) for name in ( + "attempt_capacity", "chunk_tokens", "cache_bytes", "pinned_bytes", + "output_bytes", "device_bytes", "max_workers")}} + + def _claim(self, req): + from specforge_sglang_capture.capture import request_spec + spec = request_spec(req) + if not spec or "protocol_version" not in spec: + return None + for name, expected in (("protocol_version", 1), ("server_epoch", self.epoch), + ("capture_namespace", self.namespace), + ("capture_signature", self.signature)): + if spec.get(name) != expected: + raise CaptureContractError("capture " + name + " mismatch") + if spec.get("object_key_layout") != "attempt-v1": + raise CaptureContractError("HS capture requires negotiated immutable attempt keys") + if req.sampling_params.max_new_tokens != 0: + raise CaptureContractError("HS capture requires max_new_tokens=0") + if any(getattr(req, name, None) is not None for name in ( + "multimodal_inputs", "session", "lora_id", "positional_embed_overrides", + "cache_salt", "session_id", "input_embeds")): + raise CaptureContractError("HS capture supports standard causal text only") + if set(spec.get("features", {})) != {"aux", "last_hidden"}: + raise CaptureContractError("HS capture requires complete aux and last_hidden schema") + names = list(spec["features"].values()) + for item in spec.get("passthrough", []): + if not isinstance(item, dict): + raise CaptureContractError("passthrough must contain objects") + names.append(item.get("name")) + shape = item.get("shape") + if not isinstance(shape, list) or any(type(d) is not int or d < 0 for d in shape): + raise CaptureContractError("invalid passthrough shape") + count = 1 + for dim in shape: + count *= dim + def elements(value): + return sum(elements(v) for v in value) if isinstance(value, list) else 1 + if elements(item.get("data")) != count: + raise CaptureContractError("passthrough data does not match its declared shape") + if any(not isinstance(name, str) or not name for name in names) or len(set(names)) != len(names): + raise CaptureContractError("capture output names must be distinct nonempty strings") + if spec.get("request_digest", digest_request(req.origin_input_ids, spec)) != digest_request(req.origin_input_ids, spec): + raise CaptureContractError("capture request digest mismatch") + entry = getattr(req, "_specforge_hs_attempt", None) + if entry is None: + entry = self.registry.claim(spec, req.origin_input_ids) + entry.request = req + req._specforge_hs_attempt = entry + elif (entry.revoked or entry.request is not req + or self.registry.lookup(spec) is not entry): + raise CaptureContractError("capture attempt is stale or revoked") + req.extra_key = self.namespace if self.enabled else "spec-attempt:" + entry.token + return entry + + def prepare(self, scheduler, req): + self._assert_owner() + from specforge_sglang_capture.capture import request_spec + if not (spec := request_spec(req)) or "protocol_version" not in spec: + return None + self._sync_transfers() + if self._closed or self.fatal_capture_error: + raise CaptureContractError(self.fatal_capture_error or "HS runtime is closed") + entry = self._claim(req) + execution = ExecutionIdentity(self.epoch, req.rid, entry.attempt_id, entry.execution_epoch) + if req.rid in self.candidates: + raise CaptureContractError("HS admission candidate already active") + candidate = _Candidate(execution, entry, tuple(req.origin_input_ids)) + self.candidates[req.rid] = candidate + req._specforge_hs_execution = execution + if not candidate.tokens: + raise CaptureContractError("HS capture requires a nonempty prompt") + cap = 0 + if self.writer: + candidate.lease = self.cache.lookup_and_pin( + self.namespace, candidate.tokens, + len(candidate.tokens) - 1 if self.enabled and not getattr(req, "_specforge_hs_force_full", False) else 0) + cap = candidate.lease.coverage_end + from sglang.srt.managers.prefix_admission import PrefixAdmissionProposal + return PrefixAdmissionProposal(cap, dataclasses.astuple(execution), candidate) + + def after_match(self, scheduler, req, proposal, lawful_prefix_len): + self._assert_owner() + candidate = self.candidates[req.rid] + if not 0 <= lawful_prefix_len <= len(candidate.tokens) - 1: + raise CaptureContractError("native recovery prefix is outside [0,N-1]") + if self.writer: + candidate.lease = self.cache.shrink_lease(candidate.lease, lawful_prefix_len) + candidate.L = lawful_prefix_len + + def reserve(self, scheduler, req, proposal, lawful_prefix_len): + self._assert_owner() + candidate = self.candidates[req.rid] + if candidate.L != lawful_prefix_len: + raise CaptureContractError("native recovery prefix changed after match") + if not self.writer: + return True + from specforge_sglang_capture.capture import request_spec + passthrough = 0 + from specforge_sglang_capture.sink import STR_DTYPE + for item in request_spec(req).get("passthrough", []): + dtype = STR_DTYPE.get(item.get("dtype", "int64")) + if dtype is None: + raise CaptureContractError("unsupported passthrough dtype") + shape = item.get("shape") + if not isinstance(shape, list) or any(type(d) is not int or d < 0 for d in shape): + raise CaptureContractError("invalid passthrough shape") + elements = 1 + for dim in shape: + elements *= dim + passthrough += elements * torch.empty((), dtype=dtype).element_size() + candidate.reservation = self.controller.try_reserve_output( + candidate.execution, len(candidate.tokens), lawful_prefix_len, passthrough) + return candidate.reservation is not None + + def commit(self, scheduler, req, proposal, lawful_prefix_len): + self._assert_owner() + candidate = self.candidates[req.rid] + if candidate.L != lawful_prefix_len: + raise CaptureContractError("native recovery prefix changed before commit") + if self.writer: + req._specforge_hs_plan = self.controller.freeze_plan( + candidate.execution, self.namespace, candidate.tokens, len(candidate.tokens), + lawful_prefix_len, candidate.lease, candidate.reservation) + candidate.entry.state = "ADMITTED" + candidate.entry.last_plan = {"N": len(candidate.tokens), "L": lawful_prefix_len, + "H": proposal.max_prefix_len, "span": [lawful_prefix_len, len(candidate.tokens)], + "execution_epoch": candidate.execution.execution_epoch} + self.history.append({"attempt_id": candidate.entry.attempt_id, + **candidate.entry.last_plan}) + + def rollback(self, scheduler, req, proposal, reason): + self._assert_owner() + candidate = self.candidates.pop(req.rid, None) + if candidate is None: + return + if self.writer: + if candidate.reservation is not None: + self.controller.release_output(candidate.reservation) + if candidate.lease is not None: + self.cache.release_lease(candidate.lease) + self.controller.poll_completed() + self.controller.poll_drained() + req._specforge_hs_plan = None + candidate.entry.execution_epoch += 1 + req._specforge_hs_execution = None + candidate.entry.state = "QUEUED" + + def lifecycle(self, scheduler, req, event, identity): + self._assert_owner() + if event == "shutdown": + self.shutdown() + return + if event == "weight_update": + raise CaptureContractError("HS capture forbids weight updates within a load epoch") + if req is None: + return + entry = getattr(req, "_specforge_hs_attempt", None) + if entry is None: + from specforge_sglang_capture.capture import request_spec + spec = request_spec(req) + if not spec or "protocol_version" not in spec: + return + try: + entry = self.registry.lookup(spec) + except (KeyError, ValueError): + return + entry.request = req + req._specforge_hs_attempt = entry + current = getattr(req, "_specforge_hs_execution", None) + if identity is not None and identity not in (current, dataclasses.astuple(current) if current else None): + return + if event == "finish": + return # Native finish precedes consume/deferred publication. + if entry.transfer_certificate: + return # Receiver already owns the immutable published objects. + candidate = self.candidates.pop(req.rid, None) + if candidate is not None and self.writer: + if candidate.reservation is not None: + entry.drains.append(self.controller.invalidate(candidate.execution, event)) + if candidate.lease is not None: + self.cache.release_lease(candidate.lease) + for job in tuple(self.jobs.values()): + if job.entry is entry and (current is None or job.execution == current): + self._cancel_job(job, event, revoke=event not in ("retract", "retry")) + req._specforge_hs_plan = None + req._specforge_hs_execution = None + entry.execution_epoch += 1 + if event in ("retract", "retry"): + req._specforge_hs_force_full = True + entry.state = "QUEUED" + else: + entry.revoked, entry.state = True, "DRAINING" + self._mark_drained(entry) + + def invalidate(self, entry, reason): + if entry.transfer_certificate: + return + entry.revoked, entry.state = True, "DRAINING" + if entry.request is not None: + self.lifecycle(self.s, entry.request, reason, + getattr(entry.request, "_specforge_hs_execution", None)) + self._mark_drained(entry) + + def _mark_drained(self, entry): + if (entry.state != "RETIRED" and entry.revoked and entry.deliveries == 0 + and all(handle.done() for handle in entry.drains)): + entry.state = "WRITER_DRAINED" + + def _current(self, job): + return ((not job.entry.revoked or bool(job.entry.transfer_certificate)) + and self._same_execution(job)) + + def _same_execution(self, job): + return (job.entry.execution_epoch == job.execution.execution_epoch + and job.entry.attempt_id == job.execution.attempt_id) + + def _cancel_job(self, job, reason, *, revoke=True): + if job.entry.transfer_certificate: + return + if revoke: + job.entry.revoked, job.entry.state = True, "DRAINING" + if job.failed is None: + job.failed = RuntimeError(reason) + job.entry.drains.append(self.controller.invalidate(job.execution, reason)) + + def snapshot(self, req, item, aux, last_hidden): + """Freeze compact owned suffixes on the forward stream exactly once.""" + self._assert_owner() + plan = req._specforge_hs_plan + candidate = self.candidates.pop(req.rid) + if plan.execution != candidate.execution or plan.execution in self.jobs: + raise CaptureContractError("capture plan execution is stale or already submitted") + features = {} + try: + for name, tensor in (("aux", aux), ("last_hidden", last_hidden)): + if tensor is None: + raise CaptureContractError("capture is missing " + name) + features[name] = tensor[item.start:item.start + item.length].detach().clone( + memory_format=torch.contiguous_format) + ready = None + if next(iter(features.values())).is_cuda: + ready = torch.cuda.Event() + ready.record() + future = self.controller.submit(plan, CapturedSpan( + plan.execution, plan.L, plan.N, features, ready)) + except BaseException as exc: + try: + if any(t.is_cuda for t in features.values()): + torch.cuda.current_stream(next(iter(features.values())).device).synchronize() + except BaseException as drain_error: + retained = Future() + candidate.entry.drains.append(retained) + self.quarantined_snapshots.append((plan, features, retained)) + self.fatal_capture_error = "capture snapshot drain unknown: " + str(drain_error) + raise CaptureContractError(self.fatal_capture_error) from None + candidate.entry.drains.append(self.controller.invalidate(plan.execution, "snapshot-failed")) + self.invalidate(candidate.entry, "snapshot-failed") + raise exc + output = Future() + output.set_running_or_notify_cancel() + from specforge_sglang_capture.capture import request_spec + job = _Publication(plan.execution, candidate.entry, copy.deepcopy(request_spec(req)), + plan, future, output) + self.jobs[plan.execution] = job + candidate.entry.deliveries += 1 + return job + + def consume(self, jobs, batch): + self._assert_owner() + reqs = {req.rid: req for req in batch.reqs} + for job in jobs: + req = reqs.get(job.execution.request_id) + if req is None or not self._current(job): + self._cancel_job(job, "capture request disappeared before consume") + continue + if not req.finished(): + self._cancel_job(job, "capture requests must finish at prefill") + self.capture.results[req.rid] = {"sample_id": job.spec["sample_id"], + "error": "capture requests must finish at prefill"} + continue + job.consumed = True + self.capture.defer_response(req, HSCompletion(self, job)) + self.poll() + + def capture_error(self, req, reason): + self._assert_owner() + entry = getattr(req, "_specforge_hs_attempt", None) + if entry is not None: + self.invalidate(entry, reason) + + def _drop_job(self, job): + if self.jobs.get(job.execution) is not job: + return + self.jobs.pop(job.execution) + job.entry.deliveries -= 1 + if job.entry.deliveries < 0: + raise RuntimeError("capture delivery ownership underflow") + self._mark_drained(job.entry) + + def _finish_error(self, job, error): + if self._current(job): + self._cancel_job(job, str(error)) + if job.sample is not None: + job.sample.release() + job.sample = None + if not job.output.done(): + job.output.set_exception(scalar_error(error)) + self._drop_job(job) + + def _validate_publication(self, job, result): + if not isinstance(result, dict): + raise CaptureContractError("capture sink result is not an object") + for name in ("store_id", "sample_id", "gen"): + if result.get(name) != job.spec[name]: + raise CaptureContractError("capture sink changed " + name) + names = [*job.spec["features"].values(), + *(item["name"] for item in job.spec.get("passthrough", []))] + base = (f"{job.spec['store_id']}/{job.spec['sample_id']}/g1/attempt/" + f"{job.execution.server_epoch}/{job.spec['reservation_token']}") + keys = {name: base + "/" + name for name in names} + if set(result.get("features", {})) != set(keys): + raise CaptureContractError("capture sink returned incomplete feature names") + if any(result["features"][name].get("key") != key for name, key in keys.items()): + raise CaptureContractError("capture sink immutable physical keys mismatch") + for feature in self.schema.features: + name = job.spec["features"][feature.name] + meta = result["features"][name] + if meta.get("shape") != [1, job.plan.N, feature.width] or meta.get("dtype") != feature.dtype: + raise CaptureContractError("capture sink feature schema mismatch") + for item in job.spec.get("passthrough", []): + meta = result["features"][item["name"]] + if meta.get("shape") != item["shape"] or meta.get("dtype") != item.get("dtype", "int64"): + raise CaptureContractError("capture sink passthrough schema mismatch") + if result.get("aux_layer_ids") != list(self.schema.features[0].layer_ids): + raise CaptureContractError("capture sink auxiliary layer order mismatch") + return keys + + def poll(self, scheduler=None): + """Progress independent jobs and batch only currently ready publications.""" + self._assert_owner() + if not self.writer: + return + self.controller.poll_completed() + self.controller.poll_drained() + + ready = [] + for job in tuple(self.jobs.values()): + if job.cleanup is not None: + if not job.cleanup.done(): + continue + try: + job.cleanup.result() + except Exception as exc: + # Writer has stopped reading, but deletion remains unproven. + # Keep this exact attempt live so the client can retry cleanup. + logger.error("capture cleanup failed for %s: %s", job.execution, exc) + self._finish_error(job, job.failed or RuntimeError("capture cancelled")) + continue + if job.sink is not None or not job.materialize.done(): + continue + try: + if job.sample is None: + job.sample = job.materialize.result() + if job.failed is not None or not self._current(job): + raise job.failed or RuntimeError("capture execution invalidated") + if self.fatal_capture_error: + raise RuntimeError(self.fatal_capture_error) + if job.consumed: + ready.append(job) + except Exception as exc: + self._finish_error(job, exc) + if ready: + samples = [(job.spec, job.sample.features["aux"], + job.sample.features["last_hidden"]) for job in ready] + try: + future = self.capture.sink.submit_samples(samples) + except Exception as exc: + samples.clear() + future = Future() + future.set_exception(scalar_error(exc)) + for index, job in enumerate(ready): + job.sink, job.sink_index, job.sink_count = future, index, len(ready) + job.publish_started = time.perf_counter() + job.entry.state = "PUBLISHING" + samples = None # The sink now owns and clears its argument list. + ready.clear() + for job in tuple(self.jobs.values()): + if job.sink is None or not job.sink.done() or job.cleanup is not None: + continue + try: + results = job.sink.result() + if len(results) != job.sink_count: + raise CaptureContractError("capture sink batch result count mismatch") + result = results[job.sink_index] + keys = self._validate_publication(job, result) + if job.failed is not None or not self._current(job): + job.failed = job.failed or RuntimeError("capture cancelled during publication") + # Keep the publication owner until the exact-key deletion is + # ordered after its already-completed writer. + job.cleanup = self.capture.sink.discard_sample(job.spec) + continue + entry = job.entry + if entry.deliveries != 1 or entry.readers or any(not handle.done() for handle in entry.drains): + raise CaptureContractError("capture transfer precedes all execution drains") + timings = dict(job.sample.timings) + timings["publication_queue_ms"] = (time.perf_counter() - job.publish_started) * 1000 + entry.last_plan["timings"] = timings + # Sink has cleared its source refs. Transfer allocation ownership + # to cache (or retire it), then issue the physical-key certificate. + job.sample.release() + job.sample = None + self._drop_job(job) + entry.state = "PUBLISHED" + certificate = {**self._wire(entry), "publication_ownership": "attempt-v1", + "object_key_layout": "attempt-v1", "physical_keys": keys, + "schema": self.schema_wire} + entry.transfer_certificate = {"publication_ownership": "attempt-v1", + "physical_keys": keys, "publication_certificate": certificate} + result.update(self._wire(entry)) + result.update(capture_namespace=self.namespace, capture_signature=self.signature, + reused_tokens=job.plan.L, schema=self.schema_wire) + self.registry.retire(entry) + self._transferred_tokens.append(entry.token) + if not job.output.done(): + job.output.set_result(result) + except Exception as exc: + if getattr(exc, "capture_writer_drained", True) is False: + self._cancel_job(job, "native publication drain unknown") + unknown = Future() + job.entry.drains.append(unknown) + self.quarantined_publications.append(job) + self.fatal_capture_error = "capture publication drain unknown; buffers and budgets retained" + if not job.output.done(): + job.output.set_exception(scalar_error(exc)) + self._drop_job(job) + else: + self._finish_error(job, exc) + for entry in self.registry.active.values(): + self._mark_drained(entry) + self.controller.poll_completed() + self.controller.poll_drained() + + def has_pending_work(self): + # Writer-local drain progress, like native deferred outputs. This must + # not be used as an unsynchronized TP collective idle predicate. + return bool(self.jobs or self.quarantined_snapshots or self.quarantined_publications) + + def shutdown(self): + self._assert_owner() + if self._closed: + return + self._closed = True + for entry in list(self.registry.active.values()): + self.invalidate(entry, "server-shutdown") + while self.jobs: + self.poll() + if self.jobs: + time.sleep(0.001) + if self.quarantined_snapshots or self.quarantined_publications: + raise RuntimeError("HS shutdown retains quarantined storage; native drain is unproven") + if self.controller: + self.controller.shutdown(wait=True) + self.capture.sink.close() + + def _wire(self, entry): + return {"protocol_version": 1, "server_epoch": self.epoch, "store_id": entry.identity[0], + "sample_id": entry.identity[1], "gen": 1, "attempt_id": entry.attempt_id, + "reservation_token": entry.token, "issue_sequence": entry.sequence, + "request_digest": entry.request_digest, + "cleanup_owner": entry.cleanup_owner, "cleanup_token": entry.cleanup_token, + "active_readers": len(entry.readers), + "state": entry.state, "writer_drained": entry.state in ("WRITER_DRAINED", "PUBLISHED", "RETIRED"), + **entry.last_plan, + **({"object_key_layout": entry.object_key_layout} if entry.object_key_layout else {}), + **entry.transfer_certificate} + + def control(self, scheduler, body): + self._assert_owner() + try: + return self._control(body) + except AttemptCapacityDeclined as exc: + return {"error": str(exc), "error_code": "capture_capacity_declined", + "reservation_granted": False} + except ValueError as exc: + raise CaptureControlRejected(str(exc)) from None + + def _sync_transfers(self, transferred=None, fatal=None): + if transferred is None: + pending = getattr(self, "_transferred_tokens", []) + packet = (pending, getattr(self, "fatal_capture_error", None)) if self.writer else None + transferred, fatal = self.broadcast(packet) + if fatal: + self.fatal_capture_error = fatal + if not self.writer: + for token in transferred: + entry = self.registry.tokens.get(token) + if entry is not None: + if entry.request is not None: + self.candidates.pop(entry.request.rid, None) + entry.state = "PUBLISHED" + self.registry.retire(entry) + self._transferred_tokens = [] + + def _control(self, body): + self.poll() + self._sync_transfers() + if int(body.get("protocol_version", -1)) != 1: + raise ValueError("unsupported capture protocol") + action = body.get("action") + if action == "capability": + return self.capability() + if body.get("server_epoch") != self.epoch: + raise ValueError("capture server epoch changed") + if action == "stats": + return {**self.capability(), "active_attempts": len(self.registry.active), + "pending_deliveries": len(self.jobs), "active_readers": sum(len(x.readers) for x in self.registry.active.values()), "plans": list(self.history), + "cache": self.cache.stats() if self.cache else {}, + "controller": self.controller.stats() if self.controller else {}} + if action in ("begin", "begin_batch") and getattr(self, "fatal_capture_error", None): + raise ValueError(self.fatal_capture_error) + if action == "begin_batch": + bodies = body.get("tasks") + if not isinstance(bodies, list) or not bodies or len(bodies) > self.registry.capacity: + raise ValueError("invalid begin_batch size") + if any(item.get("server_epoch") != self.epoch for item in bodies): + raise ValueError("capture server epoch changed") + tokens = self.broadcast([uuid.uuid4().hex for _ in bodies] if self.writer else None) + return {"server_epoch": self.epoch, "attempts": [self._wire(entry) for entry in self.registry.begin_batch(bodies, tokens)]} + if action == "begin": + token = self.broadcast(uuid.uuid4().hex if self.writer else None) + return self._wire(self.registry.begin(body, token)) + try: + entry = self.registry.lookup(body) + except ValueError: + entry = next((x for x in self.registry.retired if x.token == body.get("reservation_token") + and x.identity == self.registry.identity(body) and x.attempt_id == body.get("attempt_id")), None) + if entry is not None and action == "read_release": + pair = (body.get("reader_owner"), body.get("reader_token")) + if pair not in entry.released_readers: + raise ValueError("unknown retired read permit") + return dict(self._wire(entry), reader_owner=pair[0], reader_token=pair[1], reader_released=True) + if entry is not None and action in ("cleanup_acquire", "cleanup_done", "resolve"): + if action == "cleanup_done" and body.get("cleanup_token") != entry.cleanup_token: + raise ValueError("cleanup permit mismatch") + return self._wire(entry) + raise + if action == "resolve": + if body.get("cancel"): + self.invalidate(entry, "external-cancel") + if entry.request is not None: + from sglang.srt.managers.io_struct import AbortReq + self.s.abort_request(AbortReq(rid=entry.request.rid)) + self._mark_drained(entry) + wire = self.broadcast(self._wire(entry) if self.writer else None) + entry.state = wire["state"] + return wire + if action == "read_acquire": + wire = self.broadcast(self._wire(entry) if self.writer else None) + entry.state = wire["state"] + token = self.broadcast(uuid.uuid4().hex if self.writer else None) + owner = str(body.get("reader_owner") or "") + if not owner: + raise ValueError("read owner is required") + existing = entry.readers.get(owner) + # A lost acquire reply must be recoverable for release after revoke. + # This returns only an already-held token, never a new read authority. + if existing is None: + if any(previous_owner == owner for previous_owner, _ in entry.released_readers): + raise ValueError("released read owner cannot acquire again") + if len(entry.readers) + len(entry.released_readers) >= self.registry.capacity: + raise ValueError("attempt read owner history capacity exhausted") + if entry.revoked or wire["state"] != "PUBLISHED" or entry.cleanup_token is not None: + raise ValueError("attempt is not readable") + if sum(len(x.readers) for x in self.registry.active.values()) >= self.registry.capacity: + raise ValueError("read permit capacity exhausted") + entry.readers[owner] = token + return dict(self._wire(entry), reader_owner=owner, reader_token=entry.readers[owner]) + if action == "read_release": + owner, token = body.get("reader_owner"), body.get("reader_token") + if entry.readers.get(owner) == token and token is not None: + entry.readers.pop(owner) + entry.released_readers.append((owner, token)) + elif (owner, token) not in entry.released_readers: + raise ValueError("unknown read permit") + return dict(self._wire(entry), reader_owner=owner, reader_token=token, reader_released=True) + if action == "cleanup_acquire": + self._mark_drained(entry) + wire = self.broadcast(self._wire(entry) if self.writer else None) + entry.state = wire["state"] + token = self.broadcast(uuid.uuid4().hex if self.writer else None) + owner = str(body.get("cleanup_owner") or "") + if not owner or not wire["writer_drained"] or entry.readers: + raise ValueError("writer/readers not drained or cleanup owner absent") + if entry.cleanup_owner is not None and entry.cleanup_owner != owner: + raise ValueError("another store owns sample cleanup") + entry.cleanup_owner = owner + entry.cleanup_token = entry.cleanup_token or token + return self._wire(entry) + if action == "cleanup_done": + if entry.cleanup_token is None or body.get("cleanup_token") != entry.cleanup_token: + raise ValueError("cleanup permit mismatch") + self._mark_drained(entry) + wire = self.broadcast(self._wire(entry) if self.writer else None) + entry.state = wire["state"] + self.registry.retire(entry) + return self._wire(entry) + raise ValueError("unknown capture control action") diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py index 269441e40c..d211e9fabb 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py @@ -111,6 +111,25 @@ def object_key(store_id: str, sample_id: str, gen: int, name: str) -> str: return f"{store_id}/{sample_id}/g{gen}/{name}" +def sample_object_key(spec: Dict[str, Any], name: str) -> str: + if spec.get("object_key_layout") == "attempt-v1": + if not spec.get("server_epoch") or not spec.get("reservation_token"): + raise ValueError("immutable capture keys require epoch and reservation token") + return (f"{spec['store_id']}/{spec['sample_id']}/g{int(spec.get('gen', 1))}" + f"/attempt/{spec['server_epoch']}/{spec['reservation_token']}/{name}") + return object_key(str(spec["store_id"]), str(spec["sample_id"]), + int(spec.get("gen", 1)), name) + + +class CaptureWriterUncertain(RuntimeError): + """A native write may still be reading its registered source allocation.""" + capture_writer_drained = False + + +class _DrainedWriteFailure(RuntimeError): + """A complete status vector proves the native call has returned ownership.""" + + class _CudaBytes: """Exposes a raw device allocation to ``torch.as_tensor`` without copying.""" @@ -303,10 +322,14 @@ class SpecCaptureSink: def __init__(self, aux_layer_ids: Optional[Sequence[int]] = None) -> None: self.aux_layer_ids = list(aux_layer_ids) if aux_layer_ids else None self._store = None + self._owns_store = False self._put_config = None self._nvlink: Optional[NvlinkArena] = None self._control_listener: Optional[socket.socket] = None self._lock = threading.Lock() + self._uncertain_publications = [] + self._closed = False + self.require_hard_pin = False # One writer is enough: Mooncake stripes a batched transfer itself. # The thread only decouples that transfer from the scheduler. self._executor = ThreadPoolExecutor( @@ -373,10 +396,13 @@ def _connect(self): # Older ROCm Mooncake builds only expose `with_soft_pin`. if hasattr(cfg, "with_hard_pin"): cfg.with_hard_pin = True + elif self.require_hard_pin: + raise RuntimeError("HS publication requires Mooncake hard-pin ownership") elif hasattr(cfg, "with_soft_pin"): cfg.with_soft_pin = True self._put_config = cfg self._store = store + self._owns_store = True logger.info("spec-capture mooncake sink connected") return store @@ -429,12 +455,7 @@ def discard_sample(self, spec: Dict[str, Any]) -> Future: names.extend(item["name"] for item in spec.get("passthrough") or []) keys = list( dict.fromkeys( - object_key( - str(spec["store_id"]), - str(spec["sample_id"]), - int(spec.get("gen", 1)), - str(name), - ) + sample_object_key(spec, str(name)) for name in names ) ) @@ -450,6 +471,8 @@ def report_failure(done): return future def _discard_keys(self, keys: List[str]) -> None: + if any(set(keys).intersection(item["keys"]) for item in self._uncertain_publications): + raise CaptureWriterUncertain("cannot remove objects before native writer drain") if self._nvlink is not None: self._nvlink.free(keys) return @@ -486,12 +509,40 @@ def submit_samples( ``ready_event`` orders device tensors after the work that produced them. """ - return self._executor.submit(self._put_ready_samples, samples, ready_event) + if self._closed: + raise RuntimeError("capture sink is closed") + # Own a separate list: the caller may still be assigning completions + # while this worker finishes and drops its last source references. + return self._executor.submit(self._put_ready_samples, list(samples), ready_event) def _put_ready_samples(self, samples, ready_event): - if ready_event is not None: - ready_event.synchronize() - return self.put_samples(samples) + error = None + try: + if ready_event is not None: + ready_event.synchronize() + result = self.put_samples(samples) + except BaseException as exc: + # A Future retains its exception traceback. It must not retain + # drained sink tensor aliases across cache adoption. + from specforge_sglang_capture.hs.attempts import scalar_error + error = scalar_error(exc) + finally: + samples.clear() + if error is not None: + raise error + return result + + def close(self) -> None: + self._closed = True + self._executor.shutdown(wait=True, cancel_futures=False) + if self._uncertain_publications: + raise RuntimeError("capture sink retains native writes whose drain is unproven") + if self._control_listener is not None and self._nvlink is None: + self._control_listener.close() + if self._owns_store and self._store is not None: + close = getattr(self._store, "close", None) + if close is not None: + close() def put_samples(self, samples: List[Sample]) -> List[Dict[str, Any]]: """Publish samples with one batched Mooncake call; return their results. @@ -501,6 +552,8 @@ def put_samples(self, samples: List[Sample]) -> List[Dict[str, Any]]: """ if not samples: return [] + if self._uncertain_publications: + raise CaptureWriterUncertain("capture native writer is quarantined") store = None if nvlink_enabled() else self._connect() timing_enabled = os.environ.get("SGLANG_SPEC_CAPTURE_TIMING", "0") == "1" @@ -518,11 +571,11 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): tensor = tensor.detach().contiguous() else: tensor = tensor.detach().to("cpu").contiguous() - key = object_key(store_id, sample_id, gen, name) + key = sample_object_key(spec, name) keys.append(key) tensors.append(tensor) sizes.append(tensor.element_size() * tensor.numel()) - if replace: + if replace and spec.get("object_key_layout") != "attempt-v1": replace_keys.append(key) result_feats[name] = { "shape": list(tensor.shape), @@ -530,6 +583,8 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): tensor.dtype, str(tensor.dtype).replace("torch.", "") ), } + if spec.get("object_key_layout") == "attempt-v1": + result_feats[name]["key"] = key feature_meta.append(result_feats[name]) for spec, aux, last_hidden in samples: @@ -599,6 +654,7 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): self._remove_many_quiet(replace_keys) registered: List[int] = [] seen_storage = set() + submitted = quarantined = False register_started = time.perf_counter() try: for tensor, nbytes in zip(tensors, sizes): @@ -625,6 +681,7 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): register_ms = (time.perf_counter() - register_started) * 1000.0 put_started = time.perf_counter() batch_put = getattr(store, "batch_put_from", None) + submitted = True if batch_put is None: statuses = [ store.put_from(key, tensor.data_ptr(), nbytes, self._put_config) @@ -638,35 +695,34 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): self._put_config, ) put_ms = (time.perf_counter() - put_started) * 1000.0 - except Exception: + if statuses is None: + statuses = [0] * len(keys) + if len(statuses) != len(keys): + raise CaptureWriterUncertain( + f"capture sink returned {len(statuses)} statuses for {len(keys)} objects") + if any(status is not None and int(status) == -800 for status in statuses): + raise CaptureWriterUncertain("native capture transfer drain unknown (status -800)") + failed = [(key, status) for key, status in zip(keys, statuses) + if status is not None and int(status) < 0] + if failed: + raise _DrainedWriteFailure( + f"spec-capture batch_put_from failed for {len(failed)}/{len(keys)} keys; first={failed[0]}") + except Exception as exc: + if (getattr(exc, "capture_writer_drained", True) is False + or (submitted and not isinstance(exc, _DrainedWriteFailure))): + quarantined = True + self._uncertain_publications.append({"tensors": tensors, "keys": keys, + "registered": tuple(registered), "bytes": sum(sizes)}) + raise CaptureWriterUncertain(str(exc)) from None self._remove_many_quiet(keys) raise finally: - for ptr in registered: - try: - store.unregister_buffer(ptr) - except Exception: - pass - - if statuses is None: - statuses = [0] * len(keys) - if len(statuses) != len(keys): - self._remove_many_quiet(keys) - raise RuntimeError( - f"spec-capture batch_put_from returned {len(statuses)} statuses " - f"for {len(keys)} keys" - ) - failed = [ - (key, status) - for key, status in zip(keys, statuses) - if status is not None and int(status) < 0 - ] - if failed: - self._remove_many_quiet(keys) - raise RuntimeError( - f"spec-capture batch_put_from failed for {len(failed)}/{len(keys)} " - f"keys; first={failed[0]}" - ) + if not quarantined: + for ptr in registered: + try: + store.unregister_buffer(ptr) + except Exception: + pass if timing_enabled: logger.info( diff --git a/tests/test_runtime/test_hs_plugin.py b/tests/test_runtime/test_hs_plugin.py new file mode 100644 index 0000000000..b9722e1ea0 --- /dev/null +++ b/tests/test_runtime/test_hs_plugin.py @@ -0,0 +1,304 @@ +"""CPU native-output contracts; real DMA/model/TP validation remains separate.""" + +import ctypes +import dataclasses +import json +import os +import sys +import threading +import time +import unittest +from concurrent.futures import Future +from types import SimpleNamespace +from unittest import mock + +import torch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "plugins", "sglang-spec-capture")) + +from specforge_sglang_capture import capture, sink +from specforge_sglang_capture.hs.attempts import AttemptCapacityDeclined, AttemptRegistry, digest_request +from specforge_sglang_capture.hs.config import HSConfig +from specforge_sglang_capture.hs.protocol import FeatureSchema, FeatureSpec +from specforge_sglang_capture.hs.runtime import HSCaptureRuntime +from specforge.inference.capture_protocol import CaptureCapacityDeclined, CaptureProtocol + + +@dataclasses.dataclass +class _Proposal: + max_prefix_len: int + identity: tuple + opaque: object = None + + +class _Store: + def __init__(self, gate=None, status=0): + self.values, self.registered = {}, set() + self.gate, self.status = gate, status + self.entered = threading.Event() + + def register_buffer(self, ptr, size): + self.registered.add(ptr) + return 0 + + def unregister_buffer(self, ptr): + self.registered.remove(ptr) + return 0 + + def batch_put_from(self, keys, pointers, sizes, config): + self.entered.set() + if self.gate is not None and not self.gate.wait(5): + raise TimeoutError("test writer gate did not open") + for key, ptr, size in zip(keys, pointers, sizes): + self.values[key] = ctypes.string_at(ptr, size) + return [self.status] * len(keys) + + def remove(self, key, *, force=False): + self.values.pop(key, None) + return 0 + + +def _schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)), + FeatureSpec("last_hidden", 2, "float32", ()))) + + +class HSPluginTests(unittest.TestCase): + def setUp(self): + self.env = mock.patch.dict(os.environ, {"SGLANG_SPEC_CAPTURE_GPU_PUT": "0", "MOONCAKE_PROTOCOL": "tcp"}) + self.env.start() + self.protocol_module = mock.patch.dict(sys.modules, { + "sglang.srt.managers.prefix_admission": SimpleNamespace(PrefixAdmissionProposal=_Proposal)}) + self.protocol_module.start() + self.runtimes = [] + + def tearDown(self): + for runtime in self.runtimes: + if runtime.quarantined_publications or runtime.quarantined_snapshots: + runtime.capture.sink._executor.shutdown(wait=True) + runtime.controller._executor.shutdown(wait=True) + else: + runtime.shutdown() + self.protocol_module.stop() + self.env.stop() + + def runtime(self, *, store=None, cache=True, capacity=16): + writer = sink.SpecCaptureSink([1, 3]) + writer._store, writer._put_config = store or _Store(), object() + deferred = [] + native = capture.CaptureRuntime(sink=writer, is_writer=True, gpu_put=False, + max_pending_batches=2, defer_response=lambda req, completion: deferred.append((req, completion))) + scheduler = SimpleNamespace(abort_request=lambda req: None) + config = HSConfig(capture=True, cache=cache, chunk_tokens=4, cache_bytes=4096, + pinned_bytes=4096, output_bytes=8192, device_bytes=4096, attempt_capacity=capacity) + hs = HSCaptureRuntime(scheduler, native, config, + manifest={"model": "cpu-fixture", "weights": "load-epoch"}, schema=_schema(), tp_size=1) + native.hs = hs + hs.deferred = deferred + self.runtimes.append(hs) + return hs + + def request(self, hs, sample="s", *, tokens=tuple(range(8)), mask=1): + spec = {"store_id": "store", "sample_id": sample, "gen": 1, "replace": True, + "features": {"aux": "hidden_states", "last_hidden": "target"}, + "passthrough": [{"name": "loss_mask", "data": [[mask] * len(tokens)], + "shape": [1, len(tokens)], "dtype": "int64"}]} + digest = digest_request(tokens, spec) + body = {"protocol_version": 1, "action": "begin", "server_epoch": hs.epoch, + "store_id": "store", "sample_id": sample, "gen": 1, + "attempt_id": "attempt-" + sample, "request_digest": digest, + "publication_ownership": "attempt-v1"} + reply = hs.control(hs.s, body) + spec.update({key: reply[key] for key in ("server_epoch", "attempt_id", "reservation_token", "object_key_layout")}) + spec.update(protocol_version=1, capture_namespace=hs.namespace, + capture_signature=hs.signature, request_digest=digest) + return SimpleNamespace(rid="req-" + sample, origin_input_ids=list(tokens), + sampling_params=SimpleNamespace(custom_params={"spec_capture": json.dumps(spec)}, max_new_tokens=0), + finished=lambda: True) + + def admit(self, hs, req, lawful=None): + proposal = hs.prepare(hs.s, req) + L = proposal.max_prefix_len if lawful is None else lawful + hs.after_match(hs.s, req, proposal, L) + self.assertTrue(hs.reserve(hs.s, req, proposal, L)) + hs.commit(hs.s, req, proposal, L) + return proposal, L + + def forward(self, hs, reqs, lengths): + aux, last, prefixes = [], [], [] + for req, length in zip(reqs, lengths): + L = req._specforge_hs_plan.L + rows = torch.arange(L * 2, (L + length) * 2, dtype=torch.float32).view(length, 2) + aux.append(rows) + last.append(rows + 1000) + prefixes.append(L) + batch = SimpleNamespace(reqs=reqs) + forward = SimpleNamespace(extend_seq_lens_cpu=lengths, extend_prefix_lens_cpu=prefixes, + forward_mode=SimpleNamespace(is_extend=lambda: True)) + logits = SimpleNamespace(hidden_states=torch.cat(aux), last_hidden_states=torch.cat(last), + hidden_states_token_indices=None) + output = hs.capture.observer.after_forward(batch, forward, logits, can_run_graph=True) + with mock.patch.object(torch.Tensor, "to", side_effect=AssertionError("native D2H is duplicated")): + host = output.copy_to_host(lambda tensor: self.fail("HS must not call native copy_tensor")) + host.consume(batch, [None] * len(reqs)) + return output + + def spin(self, hs, condition): + deadline = time.monotonic() + 5 + while not condition(): + hs.poll() + if time.monotonic() > deadline: + self.fail("HS pipeline did not make progress") + time.sleep(0.001) + hs.poll() + + def test_native_complete_n_rows_from_nonaligned_prefix_and_loss_mask_independent_identity(self): + hs = self.runtime() + first = self.request(hs, "cold") + self.admit(hs, first) + self.forward(hs, [first], [8]) + self.spin(hs, lambda: hs.deferred[0][1].done()) + hs.deferred[0][1].result() + cold = hs.capture.results[first.rid] + self.assertEqual(cold["state"], "PUBLISHED") + self.assertTrue(cold["writer_drained"]) + self.assertEqual(cold["request_digest"], digest_request(first.origin_input_ids, capture.request_spec(first))) + second = self.request(hs, "warm", mask=0) + proposal, L = self.admit(hs, second, lawful=3) + self.assertEqual(proposal.max_prefix_len, 4) + self.assertEqual(L, 3) + self.forward(hs, [second], [5]) + self.spin(hs, lambda: hs.deferred[1][1].done()) + hs.deferred[1][1].result() + warm = hs.capture.results[second.rid] + self.assertEqual((warm["N"], warm["L"], warm["span"]), (8, 3, [3, 8])) + self.assertNotEqual(cold["request_digest"], warm["request_digest"]) + values = hs.capture.sink._store.values + for name in ("hidden_states", "target"): + self.assertEqual(values[cold["physical_keys"][name]], values[warm["physical_keys"][name]]) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + self.assertGreater(hs.cache.stats()["cache_owned_bytes"], 0) + + def test_cancel_waits_for_shared_sink_and_cannot_delete_sibling_attempt(self): + gate = threading.Event() + store = _Store(gate) + hs = self.runtime(store=store) + reqs = [self.request(hs, name) for name in ("cancel", "sibling")] + for req in reqs: + self.admit(hs, req) + self.forward(hs, reqs, [8, 8]) + self.spin(hs, store.entered.is_set) + hs.deferred[0][1].cancel() + self.assertTrue(hs.deferred[0][1].done()) + self.assertGreater(hs.controller.stats()["output_bytes"], 0) + gate.set() + self.spin(hs, lambda: not hs.jobs) + hs.deferred[1][1].result() + result = hs.capture.results[reqs[1].rid] + self.assertTrue(all(key in store.values for key in result["physical_keys"].values())) + cancelled = capture.request_spec(reqs[0]) + self.assertFalse(any(cancelled["reservation_token"] in key for key in store.values)) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + + def test_unknown_native_write_retains_sources_registration_and_output_charge(self): + store = _Store(status=-800) + hs = self.runtime(store=store) + req = self.request(hs) + self.admit(hs, req) + self.forward(hs, [req], [8]) + self.spin(hs, lambda: bool(hs.quarantined_publications)) + self.assertGreater(hs.controller.stats()["output_bytes"], 0) + self.assertTrue(store.registered) + self.assertFalse(req._specforge_hs_attempt.transfer_certificate) + self.assertTrue(hs.capture.sink._uncertain_publications) + with self.assertRaises(RuntimeError): + hs.shutdown() + + def test_cache_off_preserves_attempt_keys_full_rows_and_zero_reuse(self): + hs = self.runtime(cache=False) + for sample in ("a", "b"): + req = self.request(hs, sample) + proposal, _ = self.admit(hs, req) + self.assertEqual(proposal.max_prefix_len, 0) + self.forward(hs, [req], [8]) + self.spin(hs, lambda: hs.deferred[-1][1].done()) + hs.deferred[-1][1].result() + self.assertEqual(hs.capture.results[req.rid]["span"], [0, 8]) + self.assertFalse(hs.capability()["cache_enabled"]) + self.assertEqual(hs.cache.stats()["cache_owned_bytes"], 0) + + def test_rollback_before_and_after_freeze_releases_only_its_candidate(self): + hs = self.runtime() + req = self.request(hs) + proposal = hs.prepare(hs.s, req) + hs.after_match(hs.s, req, proposal, 0) + self.assertTrue(hs.reserve(hs.s, req, proposal, 0)) + hs.rollback(hs.s, req, proposal, "native-batch-allocation-failed") + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + self.assertEqual(hs.cache.stats()["active_leases"], 0) + next_proposal, _ = self.admit(hs, req) + self.assertGreater(next_proposal.identity[3], proposal.identity[3]) + hs.rollback(hs.s, req, next_proposal, "peer-commit-failed") + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + self.assertEqual(hs.cache.stats()["active_leases"], 0) + + def test_owner_thread_and_digest_fail_closed_before_reservation(self): + hs = self.runtime() + req = self.request(hs) + req.origin_input_ids[0] = 99 + with self.assertRaises(ValueError): + hs.prepare(hs.s, req) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + errors = [] + thread = threading.Thread(target=lambda: self._record_error(errors, hs)) + thread.start() + thread.join() + self.assertEqual(len(errors), 1) + self.assertIn("scheduler thread", str(errors[0])) + + @staticmethod + def _record_error(errors, hs): + try: + hs.control(hs.s, {"protocol_version": 1, "action": "capability"}) + except Exception as exc: + errors.append(exc) + + def test_begin_batch_capacity_is_atomic_and_exposes_only_ungranted_backpressure(self): + hs = self.runtime(capacity=2) + self.request(hs, "held") + task = lambda name: {"server_epoch": hs.epoch, "store_id": "store", "sample_id": name, + "gen": 1, "attempt_id": "a-" + name, "request_digest": "digest", + "publication_ownership": "attempt-v1"} + before = dict(hs.registry.active) + reply = hs.control(hs.s, {"protocol_version": 1, "action": "begin_batch", + "server_epoch": hs.epoch, "tasks": [task("new1"), task("new2")]}) + self.assertEqual(reply["error_code"], "capture_capacity_declined") + self.assertIs(reply["reservation_granted"], False) + self.assertEqual(hs.registry.active, before) + protocol = CaptureProtocol("http://fixture", lambda *args, **kwargs: reply, 1) + with self.assertRaises(CaptureCapacityDeclined): + protocol.call({"action": "begin_batch"}) + with self.assertRaises(RuntimeError) as failure: + protocol.call({"action": "resolve"}) + self.assertNotIsInstance(failure.exception, CaptureCapacityDeclined) + + def test_old_attempt_cleanup_never_targets_new_server_keys(self): + store = _Store() + first, second = self.runtime(store=store), self.runtime(store=store) + results = [] + specs = [] + for hs in (first, second): + req = self.request(hs, "same-sample") + specs.append(capture.request_spec(req)) + self.admit(hs, req) + self.forward(hs, [req], [8]) + self.spin(hs, lambda: hs.deferred[0][1].done()) + hs.deferred[0][1].result() + results.append(hs.capture.results[req.rid]) + first.capture.sink.discard_sample(specs[0]).result(5) + self.assertTrue(all(key in store.values for key in results[1]["physical_keys"].values())) + self.assertTrue(all(key not in store.values for key in results[0]["physical_keys"].values())) + + +if __name__ == "__main__": + unittest.main() From 37acad35368ac7ea765413f72abb34c99e46b9d4 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 02:06:10 +0800 Subject: [PATCH 06/17] fix(capture): validate full HS metadata and preserve capacity retry identity Verify request digest and exact feature names/shapes/dtypes before adopting negotiated output. Permit a proven-refused lease to coalesce with fresh registrations without reissuing its attempt identity. Managed launch explicitly sets HS switches and never inherits accidental opt-in. Use an interpreter forwarding fixture compatible with relocatable Python wrappers. Validation: CPU suite223 (221 pass/1 skip/1 expected failure); native-HS/launch101 pass. Real GPU and production validation pending. --- .../inference/adapters/server_capture.py | 34 +++++++++- specforge/launch_plan.py | 2 +- tests/test_runtime/test_capture_transfer.py | 2 + tests/test_runtime/test_launch_plan.py | 6 +- tests/test_runtime/test_native_hs_pipeline.py | 66 +++++++++++++++++++ 5 files changed, 106 insertions(+), 4 deletions(-) diff --git a/specforge/inference/adapters/server_capture.py b/specforge/inference/adapters/server_capture.py index d233eb17bf..aa8ece2e79 100644 --- a/specforge/inference/adapters/server_capture.py +++ b/specforge/inference/adapters/server_capture.py @@ -532,14 +532,20 @@ def register_prepared_batches(self, prepared): requests = [] for item in prepared: item["begin_control_url"] = self._capture_protocol.url + retryable_payloads = { + id(payload) for item in prepared if item.get("registration_refused") + for payload in item["payloads"] + } for task, payload in zip(tasks, payloads): names = [*payload["features"].values(), *(x["name"] for x in payload.get("passthrough", ()))] # Freeze the begin identity before crossing HTTP. Even a lost reply # must not let a late cleanup retarget a peer's replacement writer. - if "attempt_id" in payload and not all(item.get("registration_refused") for item in prepared): + if "attempt_id" in payload and id(payload) not in retryable_payloads: raise RuntimeError("prepared capture registration cannot be replayed") payload.update(attempt_id=payload.get("attempt_id", uuid.uuid4().hex), server_epoch=capability["server_epoch"], replace=False) + from specforge.inference.capture_protocol import request_digest + payload["request_digest"] = request_digest(task.payload["input_ids"], payload) requests.append((payload, task.payload["input_ids"], payload["attempt_id"], names)) from specforge.inference.capture_protocol import CaptureCapacityDeclined try: @@ -556,6 +562,26 @@ def register_prepared_batches(self, prepared): if fence.get("object_key_layout") == "attempt-v1": payload["object_key_layout"] = "attempt-v1" + @staticmethod + def _verify_hs_feature_metadata(result, payload, capability): + """Check the complete negotiated schema, including passthrough dtype.""" + schema = {item["name"]: item for item in capability["schema"]} + n = result.get("N") + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise RuntimeError("negotiated capture returned invalid token count") + expected = { + name: ((1, n, int(schema[artifact]["width"])), schema[artifact]["dtype"]) + for artifact, name in payload["features"].items() + } + expected.update({item["name"]: (tuple(item["shape"]), item["dtype"]) + for item in payload.get("passthrough", ())}) + actual = result.get("features", {}) + if set(actual) != set(expected): + raise RuntimeError("negotiated capture feature names mismatch") + for name, (shape, dtype) in expected.items(): + if tuple(actual[name].get("shape", ())) != shape or actual[name].get("dtype") != dtype: + raise RuntimeError("negotiated capture feature shape/dtype mismatch: " + name) + def unreturned_prepared(self, prepared, refs): """Retain only per-task failures for exact-attempt retirement.""" returned = {ref.sample_id for ref in refs} @@ -681,6 +707,10 @@ def produce_prepared(self, prepared, *, capture: CaptureConfig): if name == 'server_epoch': self._capture_protocol.invalidate_epoch(result.get(name)) raise RuntimeError("negotiated capture response identity mismatch: " + name) + if self.hs_capture: + if result.get("request_digest") != payload.get("request_digest"): + raise RuntimeError("negotiated capture response request digest mismatch") + self._verify_hs_feature_metadata(result, payload, capability) if capability.get('published_ownership') == 'attempt-v1': if result.get('publication_ownership') != 'attempt-v1' or result.get('state') != 'PUBLISHED' or not result.get('writer_drained'): raise RuntimeError('capture response omitted drained ownership transfer') @@ -691,7 +721,7 @@ def produce_prepared(self, prepared, *, capture: CaptureConfig): raise RuntimeError('capture response immutable keys mismatch') if self.hs_capture: N, L = len(task.payload["input_ids"]), result.get("L") - if not isinstance(L, int) or not 0 <= L < N or result.get("N") != N or result.get("span") != [L, N]: + if isinstance(L, bool) or not isinstance(L, int) or not 0 <= L < N or result.get("N") != N or result.get("span") != [L, N]: raise RuntimeError("negotiated capture returned invalid absolute span") ref = self._ref_from_result(task, result, capture) if capability is not None and ref.estimated_bytes > self._output_bytes([task], capability): diff --git a/specforge/launch_plan.py b/specforge/launch_plan.py index 03569cea09..28d6b8a2ad 100644 --- a/specforge/launch_plan.py +++ b/specforge/launch_plan.py @@ -772,7 +772,7 @@ def _managed_local_services( argv.extend(cfg.model.sglang_extra_args) argv.extend(server.extra_args) gpu_put = server.gpu_put if server.gpu_put is not None else default_gpu_put - hs_env = {} + hs_env = {"SPECFORGE_HS_CAPTURE": "0", "SPECFORGE_HS_CACHE": "0"} if deployment.hs_capture: if gpu_put: raise ValueError("HS capture supports host publication only; disable gpu_put") diff --git a/tests/test_runtime/test_capture_transfer.py b/tests/test_runtime/test_capture_transfer.py index a6d2f70876..cb98be100a 100644 --- a/tests/test_runtime/test_capture_transfer.py +++ b/tests/test_runtime/test_capture_transfer.py @@ -14,6 +14,8 @@ class TransferServer(ProtocolServer): def __init__(self, backend): super().__init__(backend) self.capability.update(begin_batch=True, published_ownership='attempt-v1') + for feature in self.capability['schema']: + feature['dtype'] = 'bfloat16' self.calls = [] def __call__(self, url, json_body, timeout): diff --git a/tests/test_runtime/test_launch_plan.py b/tests/test_runtime/test_launch_plan.py index b10f4baf38..5e9672bec4 100644 --- a/tests/test_runtime/test_launch_plan.py +++ b/tests/test_runtime/test_launch_plan.py @@ -7,6 +7,7 @@ import json import os import shutil +import shlex import signal import subprocess import sys @@ -2622,7 +2623,10 @@ def test_managed_preflight_checks_the_capture_python_environment(self): with tempfile.TemporaryDirectory() as parent: root = Path(parent) python = root / "capture-python" - python.symlink_to(sys.executable) + # A relocatable interpreter wrapper may resolve its prefix from + # argv[0]. Forward to its real entry point instead of moving it. + python.write_text("#!/bin/sh\nexec " + shlex.quote(sys.executable) + ' "$@"\n') + python.chmod(0o755) packages = root / "packages" for name in ( "mooncake/store", diff --git a/tests/test_runtime/test_native_hs_pipeline.py b/tests/test_runtime/test_native_hs_pipeline.py index 9e68c0fc0b..fb2895064f 100644 --- a/tests/test_runtime/test_native_hs_pipeline.py +++ b/tests/test_runtime/test_native_hs_pipeline.py @@ -38,7 +38,54 @@ def test_opt_in_rejects_missing_or_different_capability_before_begin(self): self.assertFalse(backend._d) + def test_negotiated_metadata_cannot_change_digest_dtype_or_feature_names(self): + for mutation in ("digest", "dtype", "extra"): + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + def post(url, json_body, timeout): + rows = server(url, json_body, timeout) + if url.endswith("/generate"): + result = rows[0]["meta_info"]["spec_capture"] + if mutation == "digest": + result["request_digest"] = "changed" + elif mutation == "dtype": + result["features"]["loss_mask"]["dtype"] = "float16" + else: + result["features"]["unexpected"] = dict(shape=[1, 1], dtype="int8") + return rows + adapter = SGLangServerCaptureAdapter("http://server", store, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), post_fn=post, control_fn=post, hs_capture=True, hs_prefix_cache=True) + with self.subTest(mutation=mutation), self.assertRaisesRegex(RuntimeError, "digest|shape/dtype|feature names"): + adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract()) + self.assertTrue(store._external_provisional) + store.discard_external_attempts() + self.assertFalse(backend._d) + + class CapacityBackpressure(unittest.TestCase): + def test_refused_and_new_prepared_groups_can_share_the_next_begin(self): + from specforge.inference.capture_protocol import CaptureCapacityDeclined + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + refuse = [True] + def post(url, json_body, timeout): + if json_body.get("action") == "begin_batch" and refuse[0]: + refuse[0] = False + return dict(error="full", error_code="capture_capacity_declined", reservation_granted=False) + return server(url, json_body, timeout) + adapter = SGLangServerCaptureAdapter("http://server", store, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), post_fn=post, control_fn=post, hs_capture=True, hs_prefix_cache=True) + old = adapter.prepare_refs_batches([[_task(0, 5)]], capture=_eagle3_contract()) + with self.assertRaises(CaptureCapacityDeclined): + adapter.register_prepared_batches(old) + attempt = old[0]["payloads"][0]["attempt_id"] + fresh = adapter.prepare_refs_batches([[_task(1, 5)]], capture=_eagle3_contract()) + adapter.register_prepared_batches(old + fresh) + self.assertEqual(old[0]["payloads"][0]["attempt_id"], attempt) + self.assertFalse(store._capture_pending_begin) + for group in old + fresh: + self.assertEqual(len(adapter.produce_prepared(group, capture=_eagle3_contract())), 1) + def test_atomic_refusal_retries_original_lease_without_prompt_failure(self): for concurrency in (1, 3): with self.subTest(concurrency=concurrency), tempfile.TemporaryDirectory() as root: @@ -114,3 +161,22 @@ def test_owned_transport_close_is_required_before_releasing_failed_read(self): self.assertFalse(consumer._capture_read_intents) self.assertFalse(consumer._quarantined_buffers) self.assertFalse(consumer._capture_quarantined_registrations) + + +class NativeHSLaunch(unittest.TestCase): + def test_managed_launch_exports_explicit_protocol_budget_and_keeps_guards(self): + from tests.test_runtime.test_launch_plan import _managed_raw, _build_managed + raw = _managed_raw(protocol="tcp") + config = raw["deployment"]["disaggregated"] + config.update(hs_capture=True, hs_prefix_cache=True, hs_chunk_tokens=384, hs_output_bytes=987654) + capture = _build_managed(raw).services[1].command + self.assertEqual(capture.env["SPECFORGE_HS_CAPTURE"], "1") + self.assertEqual(capture.env["SPECFORGE_HS_CACHE"], "1") + self.assertEqual(capture.env["SPECFORGE_HS_CHUNK_TOKENS"], "384") + self.assertEqual(capture.env["SPECFORGE_HS_OUTPUT_BYTES"], "987654") + self.assertEqual(capture.env["SGLANG_SPEC_CAPTURE_GPU_PUT"], "0") + self.assertIsNone(capture.env["FLASHINFER_DISABLE_VERSION_CHECK"]) + config.update(hs_capture=False, hs_prefix_cache=False) + native = _build_managed(raw).services[1].command + self.assertEqual(native.env["SPECFORGE_HS_CAPTURE"], "0") + self.assertEqual(native.env["SPECFORGE_HS_CACHE"], "0") From 097c954c77a045bc8242d628228237b407eda915 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 02:18:05 +0800 Subject: [PATCH 07/17] fix(runtime): fence new receives before transport shutdown --- .../runtime/data_plane/mooncake_store.py | 28 +++++++++- tests/test_runtime/test_native_hs_pipeline.py | 55 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/specforge/runtime/data_plane/mooncake_store.py b/specforge/runtime/data_plane/mooncake_store.py index 6ed4cac73a..4c462e0d51 100644 --- a/specforge/runtime/data_plane/mooncake_store.py +++ b/specforge/runtime/data_plane/mooncake_store.py @@ -604,6 +604,9 @@ def __init__( self.store_id = store_id or uuid.uuid4().hex[:8] self._owns_store = store is None self._closed = False + self._closing = False + self._active_get_calls = 0 + self._close_lock = threading.Lock() if store is None: kw = dict(_MOONCAKE_SETUP_DEFAULTS) kw.update(setup_kwargs or {}) @@ -1112,15 +1115,23 @@ def consumer_device(self) -> Optional[torch.device]: return torch.device("cuda", index) def close(self) -> None: + with self._close_lock: + self._close_impl() + + def _close_impl(self) -> None: """Release receive memory after loaders and removal drains have stopped. An injected/shared backend is never closed here. Its failed-transfer buffers stay quarantined because its owner may still be using transport. """ - if self._closed: - return with self._lock: - if any(intent["active"] or (intent["handle"] is not None and not intent["release_requested"]) + if self._closed: + return + # This fence is installed atomically with read admission. It stays + # closed after failed shutdown; existing reads may drain and close + # may be retried, but no new native receive can begin. + self._closing = True + if self._active_get_calls or any(intent["active"] or (intent["handle"] is not None and not intent["release_requested"]) for intent in self._capture_read_intents.values()): raise RuntimeError("cannot close capture transport with live readers") if self._receive_pool is not None: @@ -1504,6 +1515,17 @@ def _discard_external_attempts_legacy( # -- read -------------------------------------------------------------- def get(self, sample_ref: SampleRef, *, device="cpu", names=None): + with self._lock: + if self._closing or self._closed: + raise RuntimeError("Mooncake feature store is closing or closed") + self._active_get_calls += 1 + try: + return self._get_with_capture_fence(sample_ref, device=device, names=names) + finally: + with self._lock: + self._active_get_calls -= 1 + + def _get_with_capture_fence(self, sample_ref: SampleRef, *, device="cpu", names=None): self.auth.check(self._credential) self._recover_capture_reads() intent = self._begin_capture_read(sample_ref) diff --git a/tests/test_runtime/test_native_hs_pipeline.py b/tests/test_runtime/test_native_hs_pipeline.py index fb2895064f..68b5bd0bc0 100644 --- a/tests/test_runtime/test_native_hs_pipeline.py +++ b/tests/test_runtime/test_native_hs_pipeline.py @@ -5,6 +5,8 @@ """ import os import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor import unittest from unittest import mock @@ -163,6 +165,59 @@ def test_owned_transport_close_is_required_before_releasing_failed_read(self): self.assertFalse(consumer._capture_quarantined_registrations) + def test_close_fences_new_pageable_and_pooled_reads_before_transport_shutdown(self): + for kind in ("pageable", "pinned"): + with self.subTest(kind=kind): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0", receive_buffers=kind, receive_pool_bytes=4096) + consumer._owns_store = True + entered, finish = threading.Event(), threading.Event() + def close_transport(): + entered.set() + if not finish.wait(3): + raise TimeoutError("close test gate") + return 0 + with mock.patch.object(backend, "close", create=True, side_effect=close_transport), mock.patch.object(backend, "get_into", wraps=backend.get_into) as get, ThreadPoolExecutor(max_workers=1) as pool: + closed = pool.submit(consumer.close) + try: + self.assertTrue(entered.wait(2)) + with self.assertRaisesRegex(RuntimeError, "closing"): + consumer.get(ref) + self.assertEqual(get.call_count, 0) + self.assertFalse(consumer._capture_read_intents) + finally: + finish.set() + closed.result(timeout=3) + self.assertTrue(consumer._closed) + + def test_close_refuses_an_existing_receive_and_keeps_new_reads_fenced(self): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0") + consumer._owns_store = True + entered, finish = threading.Event(), threading.Event() + original = backend.get_into + def get_into(*args): + entered.set() + if not finish.wait(3): + raise TimeoutError("read test gate") + return original(*args) + with mock.patch.object(backend, "get_into", side_effect=get_into), mock.patch.object(backend, "close", create=True, return_value=0) as close, ThreadPoolExecutor(max_workers=1) as pool: + read = pool.submit(consumer.get, ref) + try: + self.assertTrue(entered.wait(2)) + with self.assertRaisesRegex(RuntimeError, "live readers"): + consumer.close() + close.assert_not_called() + with self.assertRaisesRegex(RuntimeError, "closing"): + consumer.get(ref) + finally: + finish.set() + _, handle = read.result(timeout=3) + consumer.release(handle) + consumer.close() + self.assertEqual(close.call_count, 1) + + class NativeHSLaunch(unittest.TestCase): def test_managed_launch_exports_explicit_protocol_budget_and_keeps_guards(self): from tests.test_runtime.test_launch_plan import _managed_raw, _build_managed From 9fab3946dc20f0bfd11d648f6ee89eddb6cbed5c Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 02:22:00 +0800 Subject: [PATCH 08/17] fix(capture): bound producer leases and coalescing by attempt capacity --- .../inference/adapters/server_capture.py | 22 ++++++++++ specforge/launch.py | 24 ++++++++--- tests/test_runtime/test_native_hs_pipeline.py | 40 +++++++++++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/specforge/inference/adapters/server_capture.py b/specforge/inference/adapters/server_capture.py index aa8ece2e79..3d3e4e55f8 100644 --- a/specforge/inference/adapters/server_capture.py +++ b/specforge/inference/adapters/server_capture.py @@ -518,6 +518,22 @@ def _output_bytes(self, tasks, capability): per_token += 8 return sum(len(task.payload["input_ids"]) * per_token for task in tasks) + def registration_capacity(self): + """Return the negotiated endpoint's atomic begin bound before leasing.""" + if self._capture_protocol is None: + return None + return self._registration_capacity(self._capture_protocol.capability()) + + @staticmethod + def _registration_capacity(capability): + if capability is None: + return None + capacity = capability.get("limits", {}).get("attempt_capacity") + if capacity is not None and (isinstance(capacity, bool) or + not isinstance(capacity, int) or capacity < 1): + raise RuntimeError("invalid capture attempt capacity") + return capacity + def register_prepared_batches(self, prepared): """Coalesce all currently ready leases into one endpoint begin_batch.""" if not prepared: @@ -529,6 +545,12 @@ def register_prepared_batches(self, prepared): return payloads = [payload for item in prepared for payload in item["payloads"]] tasks = [task for item in prepared for task in item["tasks"]] + capacity = self._registration_capacity(capability) + if capacity is not None and len(tasks) > capacity: + # No attempt identity or local begin intent exists yet. The producer + # splits ready groups before this boundary; direct callers receive a + # local error rather than an ambiguous partial remote reservation. + raise ValueError("registration batch exceeds capture attempt capacity") requests = [] for item in prepared: item["begin_control_url"] = self._capture_protocol.url diff --git a/specforge/launch.py b/specforge/launch.py index 3cddfcd8c7..a892a67402 100644 --- a/specforge/launch.py +++ b/specforge/launch.py @@ -1387,6 +1387,14 @@ def run_worker(w) -> None: if getattr(source, "_capture_protocol", None) is None: return run_worker_legacy(w) + # The endpoint's atomic reservation bound also limits each lease. + # Discover it before taking prompts so even a lease configured above + # the server capacity can be fulfilled without returning/failing it. + capacity_fn = getattr(source, "registration_capacity", None) + registration_capacity = capacity_fn() if callable(capacity_fn) else None + endpoint_lease = (min(worker_lease, registration_capacity) + if registration_capacity is not None else worker_lease) + # One endpoint's network registration has its own executor. The # coordinator only leases, reserves credits and harvests futures; # an in-flight begin never blocks independent ref publication. @@ -1520,7 +1528,7 @@ def clean_batch(group): record["stage"] = "generate" future = executor.submit( w.run_leased, record["tasks"], - max_tasks=worker_lease, prepared=record["prepared"], + max_tasks=endpoint_lease, prepared=record["prepared"], ) futures[future] = record @@ -1566,7 +1574,7 @@ def clean_batch(group): if accepting and not paused and submitted < round_budget and preparing is None: group = [] while len(records) < producer_concurrency and submitted < round_budget: - tasks = w.lease_tasks(worker_lease) + tasks = w.lease_tasks(endpoint_lease) if not tasks: break record = dict(tasks=tasks, start=time.perf_counter(), @@ -1580,17 +1588,23 @@ def clean_batch(group): [r["tasks"] for r in group], capture=w.capture, ), group) - # Take the entire ready snapshot. Initial idle slots - # deterministically share one begin; late arrivals queue - # behind an active registration and coalesce next turn. + # Coalesce a ready snapshot within the endpoint's atomic + # begin bound. Excess leases retain their place while + # completed siblings publish and free server capacity. budget_wait = False if (registering is None and cleaning is None and time.monotonic() >= next_registration_at): group = [] + group_tasks = 0 for record in records: if record["stage"] != "pending": continue + task_count = len(record["tasks"]) + if (registration_capacity is not None and + group_tasks + task_count > registration_capacity): + break if reserve(record): + group_tasks += task_count record["stage"] = "register" group.append(record) else: diff --git a/tests/test_runtime/test_native_hs_pipeline.py b/tests/test_runtime/test_native_hs_pipeline.py index 68b5bd0bc0..956911e705 100644 --- a/tests/test_runtime/test_native_hs_pipeline.py +++ b/tests/test_runtime/test_native_hs_pipeline.py @@ -65,6 +65,46 @@ def post(url, json_body, timeout): class CapacityBackpressure(unittest.TestCase): + def test_oversize_direct_registration_fails_before_creating_attempts(self): + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + server.capability["limits"] = {"attempt_capacity": 1} + adapter = SGLangServerCaptureAdapter("http://server", store, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), post_fn=server, control_fn=server, hs_capture=True, hs_prefix_cache=True) + prepared = adapter.prepare_refs_batches([[_task(0, 5)], [_task(1, 5)]], capture=_eagle3_contract()) + with self.assertRaisesRegex(ValueError, "attempt capacity"): + adapter.register_prepared_batches(prepared) + self.assertFalse(store._capture_pending_begin) + self.assertNotIn("begin_batch", server.calls) + self.assertTrue(all("attempt_id" not in p for item in prepared for p in item["payloads"])) + + def test_registration_and_each_lease_fit_advertised_capacity(self): + for capacity, lease, concurrency in ((1, 1, 2), (1, 3, 2), (2, 3, 3)): + with self.subTest(capacity=capacity, lease=lease, concurrency=concurrency), tempfile.TemporaryDirectory() as root: + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + server.capability["limits"] = {"attempt_capacity": capacity} + sizes = [] + def post(url, json_body, timeout): + if json_body.get("action") == "begin_batch": + sizes.append(len(json_body["tasks"])) + if sizes[-1] > capacity: + raise AssertionError("registration exceeded endpoint bound") + return server(url, json_body, timeout) + adapter = _adapter(store, post) + adapter.hs_capture = adapter.hs_prefix_cache = True + from specforge.inference.capture_protocol import CaptureProtocol + adapter._capture_protocol = CaptureProtocol(adapter.base_url, post, 1) + channel = StreamingRefChannel(os.path.join(root, "refs")) + workers, drive = _build([adapter], _prompts(4), store, channel, lease=lease, producer_concurrency=concurrency, max_worker_failures=1, max_prompt_attempts=1, backpressure_poll_s=.001, peer_wait_timeout_s=2) + self.assertEqual(drive(), 4) + self.assertEqual(workers[0].controller.status()["prompts_failed"], 0) + self.assertEqual(channel.published, 4) + self.assertTrue(sizes) + self.assertLessEqual(max(sizes), capacity) + self.assertFalse(store._capture_pending_begin) + def test_refused_and_new_prepared_groups_can_share_the_next_begin(self): from specforge.inference.capture_protocol import CaptureCapacityDeclined backend = _FakeMooncakeStore() From 68ec940b414362d17180404773a194ca76fa26f6 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 02:34:12 +0800 Subject: [PATCH 09/17] fix(capture): vote local TP failures before nested collectives Closes SF-TP-01 by synchronizing named local phases before and after internal startup, transfer, request, lookup, resolve, read and cleanup collectives. Native/HS prepare membership is agreed before transfer synchronization. Begin operations use isolated registry trials, agreed permit identities and voted reply construction before any actual grant; only unanimous uncommitted capacity declines emit typed no-grant proof. Poll and lifecycle remain free of collectives. Strengthens shared-sink cancellation to assert the same publication Future for two requests and adds actual-runtime/SG-phase threaded CPU contracts over 35 fault/success scenarios. CPU HS/core/protocol subset: 59 passed, no skips, CUDA uninitialized; real distributed/GPU/transport validation remains pending. --- plugins/sglang-spec-capture/README.md | 18 + .../specforge_sglang_capture/hs/runtime.py | 438 +++++++++++------ tests/test_runtime/test_hs_plugin.py | 13 +- tests/test_runtime/test_hs_tp_protocol.py | 450 ++++++++++++++++++ 4 files changed, 782 insertions(+), 137 deletions(-) create mode 100644 tests/test_runtime/test_hs_tp_protocol.py diff --git a/plugins/sglang-spec-capture/README.md b/plugins/sglang-spec-capture/README.md index d52c7be29c..851da0f6cb 100644 --- a/plugins/sglang-spec-capture/README.md +++ b/plugins/sglang-spec-capture/README.md @@ -241,3 +241,21 @@ proof exists. Timeouts do not authorize reuse. store; `test_hs_controller_parallel_cuda` requires actual CUDA/pinned copies. CPU fixtures do not certify numerical model, real Mooncake, scheduler TP or training equivalence. + +Internal TP control/admission collectives use named local phases: owner/poll, +writer payload construction, received-state application, target lookup and +permit validation all vote local errors before entering a later collective. +Ranks agree request membership and control branches before dispatch. Startup +epoch/model/schema/core failures use the same protocol. `poll` and `lifecycle` +themselves remain local because native completion paths may call them only on +the writer. Begin operations preflight a separate registry on every rank; +only unanimous capacity refusal of uncommitted trials can emit the typed +`reservation_granted=false` proof. A rank disagreement or partial commit fails +closed without that proof. + +`tests.test_runtime.test_hs_tp_protocol` drives the real runtime and companion +SG `PrefixAdmissionCoordinator.phase` with two CPU scheduler threads and a +blocking operation-order rendezvous. Set `SPECFORGE_HS_TEST_SG_SOURCE` to the SG +checkout when the installed editable still points at an upstream baseline. +This detects mismatched collective sequences in the tested faults; actual +distributed TP, CUDA and transport failure validation remains separate. diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py index 349c546b72..1baa3f0609 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py @@ -116,28 +116,41 @@ def __init__(self, scheduler, capture, config: HSConfig, *, manifest=None, if self.tp_size is None: from sglang.srt.runtime_context import get_parallel self.tp_size = get_parallel().tp_size - self.epoch = self.broadcast(uuid.uuid4().hex if self.writer else None) + self.epoch = self._writer_broadcast("startup.epoch", lambda: uuid.uuid4().hex) if manifest is None or schema is None: - manifest, schema = self._model_identity() - self.schema = schema - self.schema_wire = [dataclasses.asdict(feature) for feature in schema.features] - for feature in self.schema_wire: - feature["layer_ids"] = list(feature["layer_ids"]) - manifest = {**manifest, "load_epoch": self.epoch, "schema": self.schema_wire, - "chunk_tokens": config.chunk_tokens} - self.signature = hashlib.sha256(json.dumps( - manifest, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest() - self.namespace = "spec-feature-v1:" + self.signature - if len(set(self.gather(self.signature))) != 1: - raise CaptureContractError("TP capture model/schema signatures differ") - self.registry = AttemptRegistry(self.epoch, config.attempt_capacity) + manifest, schema = self._local_phase("startup.model", self._model_identity) + + def identify(): + self.schema = schema + self.schema_wire = [dataclasses.asdict(feature) for feature in schema.features] + for feature in self.schema_wire: + feature["layer_ids"] = list(feature["layer_ids"]) + identity = {**manifest, "load_epoch": self.epoch, "schema": self.schema_wire, + "chunk_tokens": config.chunk_tokens} + self.signature = hashlib.sha256(json.dumps( + identity, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest() + self.namespace = "spec-feature-v1:" + self.signature + self.registry = AttemptRegistry(self.epoch, config.attempt_capacity) + return self.signature + + signature = self._local_phase("startup.identity", identify) + self._agree("startup.signature", signature) self.cache = self.controller = None - if self.writer: - self.cache = PrefixFeatureCache(schema, config.chunk_tokens, - config.cache_bytes if self.enabled else 0) - self.controller = CaptureController( - self.cache, schema, config.pinned_bytes, config.output_bytes, - config.device_bytes, max_workers=config.max_workers) + + def create_core(): + if self.writer: + self.cache = PrefixFeatureCache(schema, config.chunk_tokens, + config.cache_bytes if self.enabled else 0) + self.controller = CaptureController( + self.cache, schema, config.pinned_bytes, config.output_bytes, + config.device_bytes, max_workers=config.max_workers) + + try: + self._local_phase("startup.core", create_core) + except Exception: + if self.controller is not None: + self.controller.shutdown(wait=True) + raise self.candidates: dict[str, _Candidate] = {} self.jobs: dict[ExecutionIdentity, _Publication] = {} self.history = deque(maxlen=256) @@ -151,6 +164,51 @@ def _assert_owner(self): if threading.get_ident() != self.owner_thread: raise RuntimeError("HS state may only be changed on the scheduler thread") + def _local_phase(self, phase, call): + """Vote local work BEFORE any subsequent internal collective. + + SG's outer callback phase runs too late to repair divergent nested + collectives. Labels also fail closed if ranks choose different plugin + branches that happen to arrive at a gather together. ``call`` must not + itself contain a collective; poll/lifecycle therefore remain local. + """ + value, error = None, None + try: + value = call() + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + votes = self.gather((phase, error)) + if any(not isinstance(vote, tuple) or len(vote) != 2 + or vote[0] != phase for vote in votes): + raise CaptureControlRejected(f"TP HS phase order differs at {phase}: {votes!r}") + if any(vote[1] is not None for vote in votes): + raise CaptureControlRejected(f"TP HS {phase} failed: {votes!r}") + return value + + def _agree(self, phase, value): + votes = self.gather((phase, value)) + if any(not isinstance(vote, tuple) or len(vote) != 2 + or vote[0] != phase or vote[1] != value for vote in votes): + raise CaptureControlRejected(f"TP HS {phase} differs: {votes!r}") + + def _writer_broadcast(self, phase, build): + payload = self._local_phase(phase + ".build", lambda: build() if self.writer else None) + return self.broadcast(payload) + + def _broadcast_wire(self, phase, entry): + wire = self._writer_broadcast(phase, lambda: self._wire(entry)) + + def apply(): + if any(wire.get(name) != expected for name, expected in ( + ("server_epoch", self.epoch), ("store_id", entry.identity[0]), + ("sample_id", entry.identity[1]), ("gen", entry.identity[2]), + ("attempt_id", entry.attempt_id), ("reservation_token", entry.token))): + raise CaptureContractError("TP HS wire identity changed") + entry.state = wire["state"] + + self._local_phase(phase + ".apply", apply) + return wire + def broadcast(self, value): if self.tp_size == 1: return value @@ -311,11 +369,20 @@ def elements(value): return entry def prepare(self, scheduler, req): - self._assert_owner() - from specforge_sglang_capture.capture import request_spec - if not (spec := request_spec(req)) or "protocol_version" not in spec: + def membership(): + self._assert_owner() + from specforge_sglang_capture.capture import request_spec + spec = request_spec(req) + return req.rid, bool(spec and "protocol_version" in spec) + + member = self._local_phase("prepare.input", membership) + self._agree("prepare.membership", member) + if not member[1]: return None - self._sync_transfers() + self._sync_transfers("prepare.transfers") + return self._local_phase("prepare.lease", lambda: self._prepare_local(req)) + + def _prepare_local(self, req): if self._closed or self.fatal_capture_error: raise CaptureContractError(self.fatal_capture_error or "HS runtime is closed") entry = self._claim(req) @@ -733,133 +800,234 @@ def _wire(self, entry): **entry.transfer_certificate} def control(self, scheduler, body): - self._assert_owner() try: return self._control(body) - except AttemptCapacityDeclined as exc: - return {"error": str(exc), "error_code": "capture_capacity_declined", - "reservation_granted": False} except ValueError as exc: raise CaptureControlRejected(str(exc)) from None - def _sync_transfers(self, transferred=None, fatal=None): - if transferred is None: - pending = getattr(self, "_transferred_tokens", []) - packet = (pending, getattr(self, "fatal_capture_error", None)) if self.writer else None - transferred, fatal = self.broadcast(packet) - if fatal: - self.fatal_capture_error = fatal - if not self.writer: - for token in transferred: - entry = self.registry.tokens.get(token) - if entry is not None: - if entry.request is not None: - self.candidates.pop(entry.request.rid, None) - entry.state = "PUBLISHED" - self.registry.retire(entry) + def _sync_transfers(self, phase): + def packet(): + return list(self._transferred_tokens), self.fatal_capture_error + + update = self._writer_broadcast(phase + ".payload", packet) + + def apply(): + transferred, fatal = update + if fatal: + self.fatal_capture_error = fatal + if not self.writer: + for token in transferred: + entry = self.registry.tokens.get(token) + if entry is not None: + if entry.request is not None: + self.candidates.pop(entry.request.rid, None) + entry.state = "PUBLISHED" + self.registry.retire(entry) + + # A peer retirement can fail after local bookkeeping or resource checks. + # Do not let its caller proceed to a later broadcast without this vote. + self._local_phase(phase + ".apply", apply) self._transferred_tokens = [] - def _control(self, body): - self.poll() - self._sync_transfers() - if int(body.get("protocol_version", -1)) != 1: + def _control_input(self, body): + self._assert_owner() + if not isinstance(body, dict) or int(body.get("protocol_version", -1)) != 1: raise ValueError("unsupported capture protocol") action = body.get("action") - if action == "capability": - return self.capability() - if body.get("server_epoch") != self.epoch: + if action not in ("capability", "stats", "begin", "begin_batch", "resolve", + "read_acquire", "read_release", "cleanup_acquire", "cleanup_done"): + raise ValueError("unknown capture control action") + if action != "capability" and body.get("server_epoch") != self.epoch: raise ValueError("capture server epoch changed") + # HTTP and typed IPC carry JSON. Agree the complete request, including + # action and fencing tokens, before choosing any nested control branch. + return action, json.dumps(body, sort_keys=True, separators=(",", ":")) + + def _control(self, body): + request = self._local_phase("control.input", lambda: self._control_input(body)) + self._agree("control.request", request) + self._local_phase("control.poll", self.poll) + self._sync_transfers("control.transfers") + action = request[0] + if action == "capability": + return self._local_phase("control.capability", self.capability) if action == "stats": - return {**self.capability(), "active_attempts": len(self.registry.active), - "pending_deliveries": len(self.jobs), "active_readers": sum(len(x.readers) for x in self.registry.active.values()), "plans": list(self.history), - "cache": self.cache.stats() if self.cache else {}, - "controller": self.controller.stats() if self.controller else {}} - if action in ("begin", "begin_batch") and getattr(self, "fatal_capture_error", None): - raise ValueError(self.fatal_capture_error) - if action == "begin_batch": - bodies = body.get("tasks") - if not isinstance(bodies, list) or not bodies or len(bodies) > self.registry.capacity: - raise ValueError("invalid begin_batch size") - if any(item.get("server_epoch") != self.epoch for item in bodies): - raise ValueError("capture server epoch changed") - tokens = self.broadcast([uuid.uuid4().hex for _ in bodies] if self.writer else None) - return {"server_epoch": self.epoch, "attempts": [self._wire(entry) for entry in self.registry.begin_batch(bodies, tokens)]} - if action == "begin": - token = self.broadcast(uuid.uuid4().hex if self.writer else None) - return self._wire(self.registry.begin(body, token)) - try: - entry = self.registry.lookup(body) - except ValueError: - entry = next((x for x in self.registry.retired if x.token == body.get("reservation_token") - and x.identity == self.registry.identity(body) and x.attempt_id == body.get("attempt_id")), None) - if entry is not None and action == "read_release": - pair = (body.get("reader_owner"), body.get("reader_token")) - if pair not in entry.released_readers: - raise ValueError("unknown retired read permit") - return dict(self._wire(entry), reader_owner=pair[0], reader_token=pair[1], reader_released=True) - if entry is not None and action in ("cleanup_acquire", "cleanup_done", "resolve"): - if action == "cleanup_done" and body.get("cleanup_token") != entry.cleanup_token: - raise ValueError("cleanup permit mismatch") - return self._wire(entry) - raise + def stats(): + return {**self.capability(), "active_attempts": len(self.registry.active), + "pending_deliveries": len(self.jobs), + "active_readers": sum(len(x.readers) for x in self.registry.active.values()), + "plans": list(self.history), + "cache": self.cache.stats() if self.cache else {}, + "controller": self.controller.stats() if self.controller else {}} + return self._local_phase("control.stats", stats) + if action in ("begin", "begin_batch"): + return self._begin_control(body, action) + + kind, entry = self._local_phase("control.lookup", lambda: self._lookup_control(body)) + key = self._local_phase("control.target-key", lambda: ( + kind, entry.identity, entry.attempt_id, entry.request_digest, entry.token, entry.sequence)) + self._agree("control.target", key) + if kind == "retired": + return self._local_phase("control.retired", lambda: self._retired_control(body, entry)) + if action == "resolve": - if body.get("cancel"): - self.invalidate(entry, "external-cancel") - if entry.request is not None: - from sglang.srt.managers.io_struct import AbortReq - self.s.abort_request(AbortReq(rid=entry.request.rid)) - self._mark_drained(entry) - wire = self.broadcast(self._wire(entry) if self.writer else None) - entry.state = wire["state"] - return wire + def resolve(): + if body.get("cancel"): + self.invalidate(entry, "external-cancel") + if entry.request is not None: + from sglang.srt.managers.io_struct import AbortReq + self.s.abort_request(AbortReq(rid=entry.request.rid)) + self._mark_drained(entry) + self._local_phase("control.resolve.local", resolve) + return self._broadcast_wire("control.resolve.wire", entry) if action == "read_acquire": - wire = self.broadcast(self._wire(entry) if self.writer else None) - entry.state = wire["state"] - token = self.broadcast(uuid.uuid4().hex if self.writer else None) - owner = str(body.get("reader_owner") or "") - if not owner: - raise ValueError("read owner is required") - existing = entry.readers.get(owner) - # A lost acquire reply must be recoverable for release after revoke. - # This returns only an already-held token, never a new read authority. - if existing is None: - if any(previous_owner == owner for previous_owner, _ in entry.released_readers): - raise ValueError("released read owner cannot acquire again") - if len(entry.readers) + len(entry.released_readers) >= self.registry.capacity: - raise ValueError("attempt read owner history capacity exhausted") - if entry.revoked or wire["state"] != "PUBLISHED" or entry.cleanup_token is not None: - raise ValueError("attempt is not readable") - if sum(len(x.readers) for x in self.registry.active.values()) >= self.registry.capacity: - raise ValueError("read permit capacity exhausted") + wire = self._broadcast_wire("control.read-acquire.wire", entry) + + def validate_read(): + owner = str(body.get("reader_owner") or "") + if not owner: + raise ValueError("read owner is required") + existing = entry.readers.get(owner) + # Lost replies can recover an existing permit after revoke; + # they cannot create a new reader after its first release. + if existing is None: + if any(old == owner for old, _ in entry.released_readers): + raise ValueError("released read owner cannot acquire again") + if len(entry.readers) + len(entry.released_readers) >= self.registry.capacity: + raise ValueError("attempt read owner history capacity exhausted") + if entry.revoked or wire["state"] != "PUBLISHED" or entry.cleanup_token is not None: + raise ValueError("attempt is not readable") + if sum(len(x.readers) for x in self.registry.active.values()) >= self.registry.capacity: + raise ValueError("read permit capacity exhausted") + return owner, existing + + owner, existing = self._local_phase("control.read-acquire.validate", validate_read) + self._agree("control.read-acquire.branch", (owner, existing)) + token = existing + if token is None: + token = self._writer_broadcast("control.read-acquire.token", lambda: uuid.uuid4().hex) + + def acquire_read(): entry.readers[owner] = token - return dict(self._wire(entry), reader_owner=owner, reader_token=entry.readers[owner]) + return dict(self._wire(entry), reader_owner=owner, reader_token=token) + return self._local_phase("control.read-acquire.commit", acquire_read) if action == "read_release": - owner, token = body.get("reader_owner"), body.get("reader_token") - if entry.readers.get(owner) == token and token is not None: - entry.readers.pop(owner) - entry.released_readers.append((owner, token)) - elif (owner, token) not in entry.released_readers: - raise ValueError("unknown read permit") - return dict(self._wire(entry), reader_owner=owner, reader_token=token, reader_released=True) + def release_read(): + owner, token = body.get("reader_owner"), body.get("reader_token") + if entry.readers.get(owner) == token and token is not None: + entry.readers.pop(owner) + entry.released_readers.append((owner, token)) + elif (owner, token) not in entry.released_readers: + raise ValueError("unknown read permit") + return dict(self._wire(entry), reader_owner=owner, reader_token=token, reader_released=True) + return self._local_phase("control.read-release", release_read) if action == "cleanup_acquire": - self._mark_drained(entry) - wire = self.broadcast(self._wire(entry) if self.writer else None) - entry.state = wire["state"] - token = self.broadcast(uuid.uuid4().hex if self.writer else None) - owner = str(body.get("cleanup_owner") or "") - if not owner or not wire["writer_drained"] or entry.readers: - raise ValueError("writer/readers not drained or cleanup owner absent") - if entry.cleanup_owner is not None and entry.cleanup_owner != owner: - raise ValueError("another store owns sample cleanup") - entry.cleanup_owner = owner - entry.cleanup_token = entry.cleanup_token or token - return self._wire(entry) + self._local_phase("control.cleanup-acquire.drain", lambda: self._mark_drained(entry)) + wire = self._broadcast_wire("control.cleanup-acquire.wire", entry) + + def validate_cleanup(): + owner = str(body.get("cleanup_owner") or "") + if not owner or not wire["writer_drained"] or entry.readers: + raise ValueError("writer/readers not drained or cleanup owner absent") + if entry.cleanup_owner is not None and entry.cleanup_owner != owner: + raise ValueError("another store owns sample cleanup") + return owner, entry.cleanup_token + + owner, existing = self._local_phase("control.cleanup-acquire.validate", validate_cleanup) + self._agree("control.cleanup-acquire.branch", (owner, existing)) + token = existing + if token is None: + token = self._writer_broadcast("control.cleanup-acquire.token", lambda: uuid.uuid4().hex) + + def acquire_cleanup(): + entry.cleanup_owner, entry.cleanup_token = owner, token + return self._wire(entry) + return self._local_phase("control.cleanup-acquire.commit", acquire_cleanup) if action == "cleanup_done": - if entry.cleanup_token is None or body.get("cleanup_token") != entry.cleanup_token: + def validate_done(): + if entry.cleanup_token is None or body.get("cleanup_token") != entry.cleanup_token: + raise ValueError("cleanup permit mismatch") + self._mark_drained(entry) + self._local_phase("control.cleanup-done.validate", validate_done) + self._broadcast_wire("control.cleanup-done.wire", entry) + + def retire(): + self.registry.retire(entry) + return self._wire(entry) + return self._local_phase("control.cleanup-done.retire", retire) + raise AssertionError("validated control action has no implementation") + + def _begin_control(self, body, action): + def validate(): + if self.fatal_capture_error: + raise ValueError(self.fatal_capture_error) + bodies = body.get("tasks") if action == "begin_batch" else [body] + if not isinstance(bodies, list) or not bodies or len(bodies) > self.registry.capacity: + raise ValueError("invalid begin_batch size") + if any(not isinstance(item, dict) or item.get("server_epoch") != self.epoch for item in bodies): + raise ValueError("capture server epoch changed") + return bodies + + bodies = self._local_phase("control.begin.validate", validate) + tokens = self._writer_broadcast("control.begin.tokens", lambda: [uuid.uuid4().hex for _ in bodies]) + prepared = self._local_phase("control.begin.preflight", lambda: self._begin_trial(bodies, tokens)) + + def proposal(): + if prepared is None: + return "capacity-declined", self.registry.capacity + _, entries = prepared + return "ready", tuple((entry.identity, entry.attempt_id, entry.request_digest, + entry.token, entry.sequence, entry.object_key_layout) for entry in entries) + + decision = self._local_phase("control.begin.proposal", proposal) + self._agree("control.begin.decision", decision) + if prepared is None: + # Every rank declined an uncommitted trial. No actual registry has + # granted anything in this operation, so this is the ONLY typed + # capacity refusal. Divergent preflights or commit failures are + # generic fail-closed errors, never proof of an ungranted mutation. + return {"error": "attempt registry capacity exhausted", + "error_code": "capture_capacity_declined", "reservation_granted": False} + trial, entries = prepared + replies = self._local_phase("control.begin.reply", lambda: [self._wire(entry) for entry in entries]) + + def commit(): + self.registry.active, self.registry.tokens, self.registry.sequence = ( + trial.active, trial.tokens, trial.sequence) + self._local_phase("control.begin.commit", commit) + return {"server_epoch": self.epoch, "attempts": replies} if action == "begin_batch" else replies[0] + + def _begin_trial(self, bodies, tokens): + trial = AttemptRegistry(self.registry.epoch, self.registry.capacity) + trial.active, trial.tokens, trial.sequence = ( + dict(self.registry.active), dict(self.registry.tokens), self.registry.sequence) + try: + entries = trial.begin_batch(bodies, tokens) + except AttemptCapacityDeclined: + return None + return trial, entries + + def _lookup_control(self, body): + try: + return "active", self.registry.lookup(body) + except ValueError: + identity = self.registry.identity(body) + entry = next((entry for entry in self.registry.retired + if entry.token == body.get("reservation_token") and entry.identity == identity + and entry.attempt_id == body.get("attempt_id")), None) + if entry is None: + raise + return "retired", entry + + def _retired_control(self, body, entry): + action = body["action"] + if action == "read_release": + pair = body.get("reader_owner"), body.get("reader_token") + if pair not in entry.released_readers: + raise ValueError("unknown retired read permit") + return dict(self._wire(entry), reader_owner=pair[0], reader_token=pair[1], reader_released=True) + if action in ("cleanup_acquire", "cleanup_done", "resolve"): + if action == "cleanup_done" and body.get("cleanup_token") != entry.cleanup_token: raise ValueError("cleanup permit mismatch") - self._mark_drained(entry) - wire = self.broadcast(self._wire(entry) if self.writer else None) - entry.state = wire["state"] - self.registry.retire(entry) return self._wire(entry) - raise ValueError("unknown capture control action") + raise ValueError("unknown, retired or mismatched reservation token") diff --git a/tests/test_runtime/test_hs_plugin.py b/tests/test_runtime/test_hs_plugin.py index b9722e1ea0..dffd4ba50a 100644 --- a/tests/test_runtime/test_hs_plugin.py +++ b/tests/test_runtime/test_hs_plugin.py @@ -124,7 +124,7 @@ def admit(self, hs, req, lawful=None): hs.commit(hs.s, req, proposal, L) return proposal, L - def forward(self, hs, reqs, lengths): + def forward(self, hs, reqs, lengths, *, wait_materialize=False): aux, last, prefixes = [], [], [] for req, length in zip(reqs, lengths): L = req._specforge_hs_plan.L @@ -140,6 +140,8 @@ def forward(self, hs, reqs, lengths): output = hs.capture.observer.after_forward(batch, forward, logits, can_run_graph=True) with mock.patch.object(torch.Tensor, "to", side_effect=AssertionError("native D2H is duplicated")): host = output.copy_to_host(lambda tensor: self.fail("HS must not call native copy_tensor")) + if wait_materialize: + self.spin(hs, lambda: all(job.materialize.done() for job in output.hs_jobs)) host.consume(batch, [None] * len(reqs)) return output @@ -186,10 +188,17 @@ def test_cancel_waits_for_shared_sink_and_cannot_delete_sibling_attempt(self): reqs = [self.request(hs, name) for name in ("cancel", "sibling")] for req in reqs: self.admit(hs, req) - self.forward(hs, reqs, [8, 8]) + output = self.forward(hs, reqs, [8, 8], wait_materialize=True) + first, sibling = output.hs_jobs + self.assertIs(first.sink, sibling.sink) + self.assertIsNotNone(first.sink) + self.assertEqual((first.sink_count, sibling.sink_count), (2, 2)) self.spin(hs, store.entered.is_set) + held = first.sample hs.deferred[0][1].cancel() self.assertTrue(hs.deferred[0][1].done()) + self.assertIs(first.sample, held) + self.assertFalse(first.sink.done()) self.assertGreater(hs.controller.stats()["output_bytes"], 0) gate.set() self.spin(hs, lambda: not hs.jobs) diff --git a/tests/test_runtime/test_hs_tp_protocol.py b/tests/test_runtime/test_hs_tp_protocol.py new file mode 100644 index 0000000000..ffd2744691 --- /dev/null +++ b/tests/test_runtime/test_hs_tp_protocol.py @@ -0,0 +1,450 @@ +"""Threaded CPU contracts for actual HS callbacks and SG's outer phase. + +The rendezvous enforces collective operation order and exchanges real Python +payloads between two scheduler-owner threads. It is not CUDA, IPC or actual TP +execution. Point SPECFORGE_HS_TEST_SG_SOURCE at the companion SG checkout when +the interpreter's editable still points at the official baseline. +""" + +import copy +import dataclasses +import importlib.util +import json +import os +from pathlib import Path +import sys +import threading +import time +from types import SimpleNamespace +import unittest +from unittest import mock + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "plugins/sglang-spec-capture")) + +from specforge_sglang_capture.capture import CaptureRuntime +from specforge_sglang_capture.hs.attempts import digest_request +from specforge_sglang_capture.hs.config import HSConfig +from specforge_sglang_capture.hs.protocol import FeatureSchema, FeatureSpec +from specforge_sglang_capture.hs.runtime import HSCaptureRuntime + + +def _sg_module(): + if source := os.environ.get("SPECFORGE_HS_TEST_SG_SOURCE"): + path = Path(source) / "python/sglang/srt/managers/prefix_admission.py" + else: + spec = importlib.util.find_spec("sglang") + if spec is None or spec.origin is None: + return None + path = Path(spec.origin).parent / "srt/managers/prefix_admission.py" + if not path.is_file(): + return None + spec = importlib.util.spec_from_file_location("_specforge_hs_test_prefix_admission", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +SG = _sg_module() + + +class _Rendezvous: + def __init__(self): + self.condition = threading.Condition() + self.steps, self.trace = {}, [[], []] + self.failed = None + + def call(self, rank, step, operation, value): + with self.condition: + if self.failed: + raise RuntimeError(self.failed) + self.trace[rank].append((operation, value[0] if operation == "gather" + and isinstance(value, tuple) and len(value) == 2 else None)) + slot = self.steps.setdefault(step, {"operation": operation, "values": {}}) + if slot["operation"] != operation or rank in slot["values"]: + self.failed = f"collective order mismatch at step {step}: {slot['operation']} / {operation}" + self.condition.notify_all() + raise RuntimeError(self.failed) + slot["values"][rank] = value + if len(slot["values"]) == 2: + slot["result"] = ([slot["values"][r] for r in (0, 1)] + if operation == "gather" else slot["values"][0]) + self.condition.notify_all() + deadline = time.monotonic() + 5 + while "result" not in slot and not self.failed: + remaining = deadline - time.monotonic() + if remaining <= 0: + self.failed = f"rank {rank} timed out at collective {step} ({operation})" + self.condition.notify_all() + break + self.condition.wait(remaining) + if self.failed: + raise RuntimeError(self.failed) + return copy.deepcopy(slot["result"]) + + +class _Sink: + aux_layer_ids = [1, 3] + + def close(self): + pass + + +def _raise(message): + raise RuntimeError(message) + + +def _schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)), + FeatureSpec("last_hidden", 2, "float32", ()))) + + +@unittest.skipIf(SG is None, "companion native SG prefix-admission module is unavailable") +class ThreadedTPProtocolTests(unittest.TestCase): + def pair(self, operation, *, initialize=None, configure=None): + rendezvous = _Rendezvous() + results, errors, snapshots = [None, None], [None, None], [None, None] + + def run(rank): + step = 0 + + def collective(operation, value): + nonlocal step + current, step = step, step + 1 + return rendezvous.call(rank, current, operation, value) + + scheduler = SimpleNamespace( + prefix_admission_broadcast=lambda value: collective("broadcast", value), + prefix_admission_gather=lambda value: collective("gather", value), + abort_request=lambda req: None) + coordinator = SG.PrefixAdmissionCoordinator.__new__(SG.PrefixAdmissionCoordinator) + coordinator.gather = scheduler.prefix_admission_gather + coordinator.broadcast = scheduler.prefix_admission_broadcast + + def phase(call): + value, failures = coordinator.phase(call) + if any(error is not None for error in failures): + return None, failures + return coordinator.broadcast(value), failures + + runtime = None + try: + native = CaptureRuntime(sink=_Sink(), is_writer=rank == 0, + gpu_put=False, max_pending_batches=2, defer_response=lambda *args: None) + config = HSConfig(capture=True, cache=True, chunk_tokens=4, + cache_bytes=256, pinned_bytes=256, output_bytes=512, + device_bytes=256, attempt_capacity=2) + kwargs = {"manifest": {"model": "threaded-cpu"}, "schema": _schema()} + if initialize is not None: + config, kwargs = initialize(rank, config, kwargs) + runtime = HSCaptureRuntime(scheduler, native, config, tp_size=2, **kwargs) + if configure is not None: + configure(rank, runtime) + results[rank] = operation(rank, runtime, phase) + except BaseException as exc: + errors[rank] = exc + finally: + if runtime is not None: + snapshots[rank] = {"samples": set(runtime.registry.active), + "tokens": set(runtime.registry.tokens)} + try: + # Cleanup is local even when a protocol failure has made + # further collectives forbidden. Do not call control. + if runtime.controller is not None: + runtime.controller.shutdown(wait=True) + except Exception as exc: + if errors[rank] is None: + errors[rank] = exc + + threads = [threading.Thread(target=run, args=(rank,), daemon=True) for rank in (0, 1)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(15) + self.assertFalse(any(thread.is_alive() for thread in threads), "scheduler CPU contract thread hung") + self.assertIsNone(rendezvous.failed) + self.assertEqual([item[0] for item in rendezvous.trace[0]], + [item[0] for item in rendezvous.trace[1]]) + self.assertEqual(len(rendezvous.trace[0]), len(rendezvous.trace[1])) + self.assertFalse(torch.cuda.is_initialized()) + return results, errors, snapshots, rendezvous.trace + + def body(self, runtime, action="begin", sample="held", **values): + return {"protocol_version": 1, "server_epoch": runtime.epoch, "action": action, + "store_id": "store", "sample_id": sample, "gen": 1, + "attempt_id": "attempt-" + sample, "request_digest": "digest", + "publication_ownership": "attempt-v1", **values} + + def begin(self, runtime, phase): + reply, failures = phase(lambda: runtime.control(runtime.s, self.body(runtime))) + self.assertFalse(any(failures)) + return reply, runtime.registry.tokens[reply["reservation_token"]] + + def failed(self, result): + values, errors, _, _ = result + self.assertEqual(errors, [None, None]) + for value, failures in values: + self.assertIsNone(value) + self.assertTrue(all(isinstance(error, str) for error in failures)) + + def test_control_poll_owner_and_writer_payload_failures_vote_before_broadcast(self): + for fault in ("owner", "poll", "transfer-payload", "capability"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + if fault == "owner" and rank == 1: + runtime._assert_owner = lambda: _raise("peer owner failure") + elif fault == "poll" and rank == 0: + runtime.controller.poll_completed = lambda: _raise("writer poll failure") + elif fault == "transfer-payload" and rank == 0: + self.fail_build(runtime, "control.transfers.payload") + elif fault == "capability" and rank == 0: + runtime.capability = lambda: _raise("writer capability failure") + return phase(lambda: runtime.control(runtime.s, + {"protocol_version": 1, "action": "capability"})) + self.failed(self.pair(operation)) + + @staticmethod + def fail_build(runtime, target): + original = runtime._writer_broadcast + + def broadcast(phase, build): + return original(phase, lambda: _raise("writer payload failure") if phase == target else build()) + runtime._writer_broadcast = broadcast + + def test_transfer_peer_retirement_failure_votes_before_next_control_or_prepare_collective(self): + for action in ("control", "prepare"): + with self.subTest(action=action): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + if rank == 0: + entry.state = "PUBLISHED" + runtime.registry.retire(entry) + runtime._transferred_tokens.append(entry.token) + else: + runtime.registry.retire = lambda entry: _raise("peer transfer apply failed") + if action == "control": + return phase(lambda: runtime.control(runtime.s, + {"protocol_version": 1, "action": "capability"})) + req = SimpleNamespace(rid="r", sampling_params=SimpleNamespace( + custom_params={"spec_capture": {"protocol_version": 1}})) + return phase(lambda: runtime.prepare(runtime.s, req)) + self.failed(self.pair(operation)) + + def test_control_lookup_and_active_retired_branch_disagreement_fail_together(self): + for fault in ("missing", "active-retired", "request-action"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + if rank == 1: + if fault == "missing": + runtime.registry.tokens.pop(entry.token) + elif fault == "active-retired": + entry.state = "PUBLISHED" + runtime.registry.retire(entry) + body = {**self.body(runtime, "resolve"), "reservation_token": reply["reservation_token"]} + if rank == 1 and fault == "request-action": + body["action"] = "capability" + return phase(lambda: runtime.control(runtime.s, body)) + self.failed(self.pair(operation)) + + def test_resolve_invalidate_abort_and_wire_errors_vote_before_later_broadcast(self): + for fault in ("invalidate", "abort", "wire"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + req = SimpleNamespace(rid="request", _specforge_hs_attempt=entry) + entry.request = req + if fault == "invalidate" and rank == 0: + runtime.invalidate = lambda *args: _raise("writer invalidation failed") + elif fault == "abort" and rank == 1: + runtime.s.abort_request = lambda req: _raise("peer abort failed") + elif fault == "wire" and rank == 0: + runtime._wire = lambda entry: _raise("writer wire failed") + body = {**self.body(runtime, "resolve"), + "reservation_token": reply["reservation_token"], "cancel": fault != "wire"} + return phase(lambda: runtime.control(runtime.s, body)) + with mock.patch.dict(sys.modules, {"sglang.srt.managers.io_struct": + SimpleNamespace(AbortReq=lambda **kwargs: SimpleNamespace(**kwargs))}): + self.failed(self.pair(operation)) + + def test_begin_preflight_never_emits_false_no_grant_on_divergence_or_partial_commit(self): + for fault in ("tokens", "preflight", "reply", "commit", "capacity-disagreement"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + if fault == "capacity-disagreement": + self.begin(runtime, phase) + self.control_success(runtime, phase, "other") + if rank == 0: + runtime.registry.active.clear() + runtime.registry.tokens.clear() + if fault == "tokens" and rank == 0: + self.fail_build(runtime, "control.begin.tokens") + elif fault == "preflight" and rank == 1: + runtime._begin_trial = lambda *args: _raise("peer preflight failed") + elif fault == "reply" and rank == 0: + runtime._wire = lambda entry: _raise("writer reply failed") + elif fault == "commit" and rank == 1: + original = runtime._local_phase + def local(phase, call): + return original(phase, lambda: _raise("peer commit failed") + if phase == "control.begin.commit" else call()) + runtime._local_phase = local + return phase(lambda: runtime.control(runtime.s, self.body(runtime, sample="new"))) + result = self.pair(operation) + self.failed(result) + snapshots = result[2] + new = ("store", "new", 1) + if fault == "commit": + self.assertIn(new, snapshots[0]["samples"]) + self.assertNotIn(new, snapshots[1]["samples"]) + else: + self.assertTrue(all(new not in snapshot["samples"] for snapshot in snapshots)) + + def control_success(self, runtime, phase, sample): + reply, failures = phase(lambda: runtime.control(runtime.s, self.body(runtime, sample=sample))) + self.assertFalse(any(failures)) + return reply + + def test_unanimous_capacity_decline_is_atomic_and_typed(self): + def operation(rank, runtime, phase): + self.begin(runtime, phase) + self.control_success(runtime, phase, "other") + return phase(lambda: runtime.control(runtime.s, self.body(runtime, sample="new"))) + values, errors, snapshots, _ = self.pair(operation) + self.assertEqual(errors, [None, None]) + for reply, failures in values: + self.assertFalse(any(failures)) + self.assertEqual(reply["error_code"], "capture_capacity_declined") + self.assertIs(reply["reservation_granted"], False) + self.assertTrue(all(len(snapshot["samples"]) == 2 for snapshot in snapshots)) + + def test_read_cleanup_validation_and_existing_permit_branches_are_voted(self): + for fault in ("read-validate", "read-branch", "read-token", "cleanup-validate", + "cleanup-branch", "cleanup-token", "cleanup-done", "cleanup-retire", "retired-read"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + body = {**self.body(runtime), "reservation_token": reply["reservation_token"]} + if fault.startswith("read"): + entry.state = "PUBLISHED" + body.update(action="read_acquire", reader_owner="owner") + if fault == "read-validate" and rank == 1: + entry.released_readers.append(("owner", "released-token")) + elif fault == "read-branch" and rank == 1: + entry.readers["owner"] = "existing-token" + elif fault == "read-token" and rank == 0: + self.fail_build(runtime, "control.read-acquire.token") + elif fault == "retired-read": + entry.state = "PUBLISHED" + runtime.registry.retire(entry) + if rank == 0: + entry.released_readers.append(("owner", "released-token")) + body.update(action="read_release", reader_owner="owner", reader_token="released-token") + else: + entry.revoked, entry.state = True, "WRITER_DRAINED" + body.update(action="cleanup_acquire", cleanup_owner="owner") + if fault == "cleanup-validate" and rank == 1: + entry.readers["reader"] = "reader-token" + elif fault == "cleanup-branch" and rank == 1: + entry.cleanup_owner, entry.cleanup_token = "owner", "existing-token" + elif fault == "cleanup-token" and rank == 0: + self.fail_build(runtime, "control.cleanup-acquire.token") + elif fault in ("cleanup-done", "cleanup-retire"): + entry.cleanup_owner, entry.cleanup_token = "owner", "cleanup-token" + body.update(action="cleanup_done", cleanup_token="cleanup-token") + if fault == "cleanup-done" and rank == 1: + entry.cleanup_token = "different-token" + elif fault == "cleanup-retire" and rank == 1: + runtime.registry.retire = lambda entry: _raise("peer retirement failed") + return phase(lambda: runtime.control(runtime.s, body)) + self.failed(self.pair(operation)) + + def test_prepare_agrees_membership_and_votes_local_parse_and_lease_failures(self): + for fault in ("membership", "parse", "lease"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + tokens = list(range(8)) + spec = {"features": {"aux": "aux", "last_hidden": "last"}, "passthrough": []} + body = self.body(runtime, request_digest=digest_request(tokens, spec)) + reply, failures = phase(lambda: runtime.control(runtime.s, body)) + self.assertFalse(any(failures)) + spec.update({key: reply[key] for key in ( + "server_epoch", "attempt_id", "reservation_token", "object_key_layout", + "store_id", "sample_id", "gen", "request_digest")}) + spec.update(protocol_version=1, capture_namespace=runtime.namespace, + capture_signature=runtime.signature) + req = SimpleNamespace(rid="request", origin_input_ids=tokens, + sampling_params=SimpleNamespace(custom_params={"spec_capture": spec}, max_new_tokens=0)) + if fault == "membership" and rank == 1: + req.sampling_params.custom_params = None + elif fault == "parse" and rank == 1: + req.sampling_params = None + elif fault == "lease" and rank == 0: + runtime.cache.lookup_and_pin = lambda *args: _raise("writer lease lookup failed") + outcome = phase(lambda: runtime.prepare(runtime.s, req)) + runtime.rollback(runtime.s, req, None, "CPU contract teardown") + return outcome + self.failed(self.pair(operation)) + + def test_startup_epoch_model_identity_and_core_failures_terminate_both_ranks(self): + for fault in ("epoch", "model", "identity", "core"): + with self.subTest(fault=fault): + original = HSCaptureRuntime._writer_broadcast + def broadcast(runtime, phase, build): + return original(runtime, phase, lambda: _raise("writer epoch failure") + if fault == "epoch" and phase == "startup.epoch" else build()) + def initialize(rank, config, kwargs): + if fault == "model": + kwargs = {"manifest": None, "schema": None} + elif fault == "identity" and rank == 1: + kwargs["schema"] = None + # Keep the identity-build failure after model resolution. + kwargs["schema"] = SimpleNamespace(features=(object(),)) + elif fault == "core" and rank == 0: + config = dataclasses.replace(config, max_workers=0) + return config, kwargs + def model(runtime): + if not runtime.writer: + _raise("peer backend capability failure") + return {"model": "threaded-cpu"}, _schema() + with mock.patch.object(HSCaptureRuntime, "_writer_broadcast", broadcast), \ + mock.patch.object(HSCaptureRuntime, "_model_identity", model): + values, errors, _, _ = self.pair( + lambda rank, runtime, phase: self.fail("failed startup returned a runtime"), + initialize=initialize) + self.assertEqual(values, [None, None]) + self.assertTrue(all(error is not None for error in errors)) + + def test_successful_permit_lifecycle_and_native_prepare_keep_phase_order(self): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + wire = {**self.body(runtime), "reservation_token": reply["reservation_token"]} + entry.state = "PUBLISHED" + read, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "read_acquire", "reader_owner": "reader"})) + self.assertFalse(any(failures)) + released, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "read_release", "reader_owner": "reader", "reader_token": read["reader_token"]})) + self.assertFalse(any(failures)) + entry.revoked, entry.state = True, "WRITER_DRAINED" + cleanup, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "cleanup_acquire", "cleanup_owner": "cleanup"})) + self.assertFalse(any(failures)) + retired, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "cleanup_done", "cleanup_token": cleanup["cleanup_token"]})) + self.assertFalse(any(failures)) + req = SimpleNamespace(rid="native", sampling_params=SimpleNamespace(custom_params=None)) + value, failures = phase(lambda: runtime.prepare(runtime.s, req)) + self.assertFalse(any(failures)) + self.assertIsNone(value) + return retired + values, errors, snapshots, _ = self.pair(operation) + self.assertEqual(errors, [None, None]) + self.assertTrue(all(value["state"] == "RETIRED" for value in values)) + self.assertTrue(all(not snapshot["tokens"] for snapshot in snapshots)) + + +if __name__ == "__main__": + unittest.main() From 7cfa9b6b7d3048695457d396e9a4f9ebe3c4c0be Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 02:57:50 +0800 Subject: [PATCH 10/17] fix(capture): retain resolved TP rank in HS publication diagnostics Snapshot tp_rank alongside tp_size from SGLang runtime_context.get_parallel, the topology used by native capture writer election. Preserve the local rank in each committed plan so history, PUBLISHED results and transfer certificates retain the required diagnostic without changing physical identity or ownership. Tests check the resolved-context path, cache-off/result/certificate fields and separate rank0/rank1 committed diagnostics without a scheduler.tp_rank attribute. Relevant CPU subset: 9 passed, no skips, CUDA uninitialized; actual GPU validation remains with the validation owner. --- .../specforge_sglang_capture/hs/runtime.py | 11 ++++--- tests/test_runtime/test_hs_plugin.py | 17 ++++++++--- tests/test_runtime/test_hs_tp_protocol.py | 30 ++++++++++++++++++- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py index 1baa3f0609..1ca13eb18f 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py @@ -107,15 +107,18 @@ class HSCaptureRuntime: control_namespace = "specforge_hs" def __init__(self, scheduler, capture, config: HSConfig, *, manifest=None, - schema=None, tp_size=None): + schema=None, tp_size=None, tp_rank=0): self.s, self.capture, self.config = scheduler, capture, config self.capture.sink.require_hard_pin = True self.writer, self.enabled = capture.is_writer, config.cache self.owner_thread = threading.get_ident() - self.tp_size = tp_size + self.tp_size, self.tp_rank = tp_size, tp_rank if self.tp_size is None: from sglang.srt.runtime_context import get_parallel - self.tp_size = get_parallel().tp_size + parallel = get_parallel() + # Native capture elects the attention-TP writer from this same + # resolved topology; keep its TP rank as execution diagnostics. + self.tp_size, self.tp_rank = parallel.tp_size, parallel.tp_rank self.epoch = self._writer_broadcast("startup.epoch", lambda: uuid.uuid4().hex) if manifest is None or schema is None: manifest, schema = self._local_phase("startup.model", self._model_identity) @@ -449,7 +452,7 @@ def commit(self, scheduler, req, proposal, lawful_prefix_len): candidate.entry.state = "ADMITTED" candidate.entry.last_plan = {"N": len(candidate.tokens), "L": lawful_prefix_len, "H": proposal.max_prefix_len, "span": [lawful_prefix_len, len(candidate.tokens)], - "execution_epoch": candidate.execution.execution_epoch} + "execution_epoch": candidate.execution.execution_epoch, "rank": self.tp_rank} self.history.append({"attempt_id": candidate.entry.attempt_id, **candidate.entry.last_plan}) diff --git a/tests/test_runtime/test_hs_plugin.py b/tests/test_runtime/test_hs_plugin.py index dffd4ba50a..f901b314e7 100644 --- a/tests/test_runtime/test_hs_plugin.py +++ b/tests/test_runtime/test_hs_plugin.py @@ -82,7 +82,7 @@ def tearDown(self): self.protocol_module.stop() self.env.stop() - def runtime(self, *, store=None, cache=True, capacity=16): + def runtime(self, *, store=None, cache=True, capacity=16, parallel=None): writer = sink.SpecCaptureSink([1, 3]) writer._store, writer._put_config = store or _Store(), object() deferred = [] @@ -91,8 +91,14 @@ def runtime(self, *, store=None, cache=True, capacity=16): scheduler = SimpleNamespace(abort_request=lambda req: None) config = HSConfig(capture=True, cache=cache, chunk_tokens=4, cache_bytes=4096, pinned_bytes=4096, output_bytes=8192, device_bytes=4096, attempt_capacity=capacity) - hs = HSCaptureRuntime(scheduler, native, config, - manifest={"model": "cpu-fixture", "weights": "load-epoch"}, schema=_schema(), tp_size=1) + kwargs = dict(manifest={"model": "cpu-fixture", "weights": "load-epoch"}, schema=_schema()) + if parallel is None: + hs = HSCaptureRuntime(scheduler, native, config, tp_size=1, **kwargs) + else: + getter = mock.Mock(return_value=parallel) + with mock.patch.dict(sys.modules, {"sglang.srt.runtime_context": SimpleNamespace(get_parallel=getter)}): + hs = HSCaptureRuntime(scheduler, native, config, **kwargs) + getter.assert_called_once_with() native.hs = hs hs.deferred = deferred self.runtimes.append(hs) @@ -155,7 +161,7 @@ def spin(self, hs, condition): hs.poll() def test_native_complete_n_rows_from_nonaligned_prefix_and_loss_mask_independent_identity(self): - hs = self.runtime() + hs = self.runtime(parallel=SimpleNamespace(tp_size=1, tp_rank=0)) first = self.request(hs, "cold") self.admit(hs, first) self.forward(hs, [first], [8]) @@ -163,6 +169,8 @@ def test_native_complete_n_rows_from_nonaligned_prefix_and_loss_mask_independent hs.deferred[0][1].result() cold = hs.capture.results[first.rid] self.assertEqual(cold["state"], "PUBLISHED") + self.assertEqual(cold["rank"], hs.tp_rank) + self.assertEqual(cold["publication_certificate"]["rank"], hs.tp_rank) self.assertTrue(cold["writer_drained"]) self.assertEqual(cold["request_digest"], digest_request(first.origin_input_ids, capture.request_spec(first))) second = self.request(hs, "warm", mask=0) @@ -233,6 +241,7 @@ def test_cache_off_preserves_attempt_keys_full_rows_and_zero_reuse(self): self.spin(hs, lambda: hs.deferred[-1][1].done()) hs.deferred[-1][1].result() self.assertEqual(hs.capture.results[req.rid]["span"], [0, 8]) + self.assertEqual(hs.capture.results[req.rid]["rank"], hs.tp_rank) self.assertFalse(hs.capability()["cache_enabled"]) self.assertEqual(hs.cache.stats()["cache_owned_bytes"], 0) diff --git a/tests/test_runtime/test_hs_tp_protocol.py b/tests/test_runtime/test_hs_tp_protocol.py index ffd2744691..fa0ba7f791 100644 --- a/tests/test_runtime/test_hs_tp_protocol.py +++ b/tests/test_runtime/test_hs_tp_protocol.py @@ -139,7 +139,7 @@ def phase(call): kwargs = {"manifest": {"model": "threaded-cpu"}, "schema": _schema()} if initialize is not None: config, kwargs = initialize(rank, config, kwargs) - runtime = HSCaptureRuntime(scheduler, native, config, tp_size=2, **kwargs) + runtime = HSCaptureRuntime(scheduler, native, config, tp_size=2, tp_rank=rank, **kwargs) if configure is not None: configure(rank, runtime) results[rank] = operation(rank, runtime, phase) @@ -445,6 +445,34 @@ def operation(rank, runtime, phase): self.assertTrue(all(value["state"] == "RETIRED" for value in values)) self.assertTrue(all(not snapshot["tokens"] for snapshot in snapshots)) + def test_committed_diagnostics_keep_each_tp_rank_without_scheduler_rank_attribute(self): + def operation(rank, runtime, phase): + tokens = list(range(8)) + spec = {"features": {"aux": "aux", "last_hidden": "last"}, "passthrough": []} + body = self.body(runtime, request_digest=digest_request(tokens, spec)) + reply, failures = phase(lambda: runtime.control(runtime.s, body)) + self.assertFalse(any(failures)) + spec.update({key: reply[key] for key in ( + "server_epoch", "attempt_id", "reservation_token", "object_key_layout", + "store_id", "sample_id", "gen", "request_digest")}) + spec.update(protocol_version=1, capture_namespace=runtime.namespace, + capture_signature=runtime.signature) + req = SimpleNamespace(rid="request", origin_input_ids=tokens, + sampling_params=SimpleNamespace(custom_params={"spec_capture": spec}, max_new_tokens=0)) + proposal, failures = phase(lambda: runtime.prepare(runtime.s, req)) + self.assertFalse(any(failures)) + # The helper broadcasts the writer's proposal. Each rank's opaque + # candidate remains local, as it does in actual SG coordination. + phase(lambda: runtime.after_match(runtime.s, req, proposal, 0)) + phase(lambda: runtime.reserve(runtime.s, req, proposal, 0)) + phase(lambda: runtime.commit(runtime.s, req, proposal, 0)) + self.assertFalse(hasattr(runtime.s, "tp_rank")) + return req._specforge_hs_attempt.last_plan["rank"] + + values, errors, _, _ = self.pair(operation) + self.assertEqual(errors, [None, None]) + self.assertEqual(values, [0, 1]) + if __name__ == "__main__": unittest.main() From 0aa4821771086a4791675f557ec49aaef93cccb7 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 03:05:34 +0800 Subject: [PATCH 11/17] fix(capture): gate attention DP on resolved topology at startup Use ParallelContext.attn_dp_size instead of the deprecated enable_dp_attention record field, which remains false after SG resolution. Replica DP, PP and CP startup gates are unchanged. The new CPU regression invokes the actual _model_identity with the real get_parallel/ParallelContext override: attention-DP2 plus legacy false is rejected before checkpoint/backend validation; ordinary TP1/TP2 attention-DP1 and existing replica/PP/CP boundaries are checked. Six focused gate/rank/publication contracts passed with CUDA uninitialized; no GPU, environment or integration checkout changes. --- .../specforge_sglang_capture/hs/runtime.py | 3 +- tests/test_runtime/test_hs_topology.py | 90 +++++++++++++++++++ 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 tests/test_runtime/test_hs_topology.py diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py index 1ca13eb18f..ed2b794fa5 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py @@ -245,7 +245,8 @@ def _model_identity(self): forbidden = { "chunked_prefill": schedule.chunked_prefill_size != -1, "PP": parallel.pp_size != 1, - "DP": parallel.dp_size != 1 or bool(args.enable_dp_attention), + "DP": parallel.dp_size != 1, + "attention_DP": parallel.attn_dp_size != 1, "CP": parallel.attn_cp_size != 1 or parallel.dcp_size != 1, "HiCache": bool(args.enable_hierarchical_cache), "LoRA": bool(args.enable_lora), diff --git a/tests/test_runtime/test_hs_topology.py b/tests/test_runtime/test_hs_topology.py new file mode 100644 index 0000000000..e57928ba72 --- /dev/null +++ b/tests/test_runtime/test_hs_topology.py @@ -0,0 +1,90 @@ +"""CPU startup topology contracts using the real SG ParallelContext getter.""" + +import os +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest import mock + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "plugins/sglang-spec-capture")) + +from sglang.srt import runtime_context +from sglang.srt.arg_groups.model_override_base import resolved_view +from sglang.srt.managers import prefix_admission +from specforge_sglang_capture.hs.config import HSConfig +from specforge_sglang_capture.hs.protocol import CaptureContractError +from specforge_sglang_capture.hs.runtime import HSCaptureRuntime + + +class HSStartupTopologyTests(unittest.TestCase): + def setUp(self): + self.args = SimpleNamespace( + enable_dp_attention=False, enable_hierarchical_cache=False, + enable_lora=False, enable_session_radix_cache=False, + disable_radix_cache=False, enable_memory_saver=False, + enable_deterministic_inference=True, model_path="/cpu-fixture", + revision=None, quantization=None, load_format="auto", kv_cache_dtype="auto", + bf16_gemm_backend="auto", attention_backend="flashinfer", prefill_attention_backend=None) + text = SimpleNamespace(hidden_size=4, to_dict=lambda: {"hidden_size": 4}) + runtime = HSCaptureRuntime.__new__(HSCaptureRuntime) + runtime.s = SimpleNamespace(server_args=self.args, enable_priority_preemption=False, + prefill_delayer=None, dllm_config=None, spec_algorithm=SimpleNamespace(is_none=lambda: True), + page_size=1, model_config=SimpleNamespace(hf_text_config=text, dtype=torch.float32)) + runtime.config, runtime.enabled = HSConfig(capture=True, cache=True, chunk_tokens=4), True + self.runtime = runtime + self.parallel = runtime_context.get_parallel() + self.assertIsInstance(self.parallel, runtime_context.ParallelContext) + self.env = mock.patch.dict(os.environ, {"MOONCAKE_PROTOCOL": "tcp", "SGLANG_SPEC_CAPTURE_GPU_PUT": "0"}) + self.env.start() + self.addCleanup(self.env.stop) + + def identity(self, **overrides): + topology = dict(tp_size=2, tp_rank=0, pp_size=1, pp_rank=0, dp_size=1, + attn_dp_size=1, attn_dp_rank=0, attn_cp_size=1, attn_cp_rank=0, + attn_tp_size=2, attn_tp_rank=0, dcp_size=1) + topology.update(overrides) + features = SimpleNamespace(return_aux_hidden_state_layer_ids=[1, 3], aux_hidden_state_capture="dflash") + with self.parallel.override(**topology), \ + mock.patch.object(runtime_context, "get_schedule", return_value=SimpleNamespace(chunked_prefill_size=-1)), \ + mock.patch.object(runtime_context, "get_exec", return_value=SimpleNamespace(features=features)), \ + mock.patch.object(prefix_admission, "validate_capture_checkpoint_support", + return_value={"backend": "isolated-cpu-fixture"}) as checkpoint: + # get_parallel itself is real. The unrelated numerical-backend gate + # is isolated; unsupported topology must fail before invoking it. + try: + return self.runtime._model_identity(), checkpoint.call_count + finally: + self.checkpoint_calls = checkpoint.call_count + + def test_attention_dp_is_rejected_with_false_deprecated_flag(self): + self.assertIs(resolved_view(self.args).enable_dp_attention, False) + with self.assertRaisesRegex(CaptureContractError, "attention_DP"): + self.identity(attn_dp_size=2, attn_tp_size=1) + self.assertEqual(self.checkpoint_calls, 0) + self.assertFalse(torch.cuda.is_initialized()) + + def test_ordinary_tp1_and_tp2_attention_dp1_keep_startup_identity(self): + for width in (1, 2): + with self.subTest(tp_size=width): + (manifest, schema), checkpoint_calls = self.identity(tp_size=width, attn_tp_size=width) + self.assertEqual(manifest["tp_size"], width) + self.assertEqual([feature.name for feature in schema.features], ["aux", "last_hidden"]) + self.assertEqual(checkpoint_calls, 1) + self.assertFalse(torch.cuda.is_initialized()) + + def test_replica_dp_pp_and_cp_startup_rejections_remain(self): + for topology, diagnostic in ((dict(dp_size=2), "DP"), (dict(pp_size=2), "PP"), + (dict(attn_cp_size=2, attn_tp_size=1), "CP"), + (dict(dcp_size=2), "CP")): + with self.subTest(topology=topology): + with self.assertRaisesRegex(CaptureContractError, diagnostic): + self.identity(**topology) + self.assertEqual(self.checkpoint_calls, 0) + self.assertFalse(torch.cuda.is_initialized()) + + +if __name__ == "__main__": + unittest.main() From 5b37de5a88628e090e7ca9f21ae649948596091b Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 17:58:14 +0800 Subject: [PATCH 12/17] perf(hs): skip disabled-cache adoption and empty-lease hashes --- .../specforge_sglang_capture/hs/cache.py | 14 +- .../specforge_sglang_capture/hs/controller.py | 12 +- tests/test_runtime/test_hs_unused_work.py | 124 ++++++++++++++++++ 3 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 tests/test_runtime/test_hs_unused_work.py diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py index 99da91219f..63d300535a 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import os import struct import threading import uuid @@ -68,6 +69,9 @@ def __init__(self, schema: FeatureSchema, chunk_tokens: int, capacity_bytes: int self.schema = schema self.chunk_tokens = checked_int(chunk_tokens, "chunk_tokens", 1) self.capacity_bytes = checked_int(capacity_bytes, "capacity_bytes") + # Freeze this policy for the lifetime of a cache/controller pair. Setting + # the flag to 0 retains the old work for paired diagnostics and rollback. + self.skip_unused_cache_work = os.environ.get("SPECFORGE_HS_SKIP_UNUSED_CACHE_WORK", "1") != "0" self._chunk_bytes = self.chunk_tokens * self.schema.row_bytes self._lock = threading.RLock() self._entries: dict[ContentBlockKey, _Entry] = {} @@ -139,7 +143,13 @@ def _complete(self, entry: _Entry) -> bool: def lookup_and_pin(self, namespace: str, tokens: tuple[int, ...], limit: int) -> PrefixLease: _token_tuple(tokens) checked_int(limit, "coverage limit") - keys = self.block_keys(namespace, tokens[: min(len(tokens), limit)]) + if self.skip_unused_cache_work and self.capacity_bytes == 0: + # A disabled cache cannot have content. Still validate the external + # namespace/tokens and issue a real, identity-checked empty lease. + self._root_digest(namespace) + keys = () + else: + keys = self.block_keys(namespace, tokens[: min(len(tokens), limit)]) with self._lock: entries = [] for key in keys: @@ -169,6 +179,8 @@ def _checked_lease(self, lease: PrefixLease, end: int, if namespace is not None and namespace != rec.namespace: raise CaptureContractError("lease namespace differs from plan") if tokens is not None: + _token_tuple(tokens) + if tokens is not None and (end or not self.skip_unused_cache_work): expected = self.block_keys(rec.namespace, tokens) needed = (end + self.chunk_tokens - 1) // self.chunk_tokens if tuple(e.key for e in rec.entries[:needed]) != expected[:needed]: diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py index 46ff01cae1..2b4dca0d52 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py @@ -116,6 +116,7 @@ def __init__(self, cache: PrefixFeatureCache, schema: FeatureSchema, "admission_declined_fragmentation_attempts", ), 0) self._adoption_calls = 0 + self._adoption_unused_skips = 0 self._adoption_total_ms = self._adoption_max_ms = 0.0 self._executor = ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="spec-feature-capture") @@ -316,8 +317,11 @@ def _sample_released(self, reservation: OutputReservation) -> None: # after this ownership transfer. Keep the output charge until # the cache has charged every retained backing allocation. if owned and not state.cancelled and not rec.release_requested and not self._closed: - self._adoption_calls += 1 - self.cache._adopt_released_sample(state.cache_keys, owned) + if state.cache_keys or not self.cache.skip_unused_cache_work: + self._adoption_calls += 1 + self.cache._adopt_released_sample(state.cache_keys, owned) + else: + self._adoption_unused_skips += 1 except Exception: self._fill_failures += 1 finally: @@ -439,6 +443,9 @@ def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: def _prepare_cache_adoption(self, job: _Job) -> tuple: # Hash/construct keys on the assembly worker, never in the sink release # callback (which may run on the scheduler thread). + if self.cache.skip_unused_cache_work and ( + self.cache.capacity_bytes == 0 or job.plan.N < self.cache.chunk_tokens): + return () try: return self.cache.block_keys(job.plan.namespace, job.plan.tokens) except Exception: @@ -581,6 +588,7 @@ def stats(self) -> dict[str, int | float]: fill_failures=self._fill_failures, capture_workers=self._max_workers, cache_adoption_calls=self._adoption_calls, + cache_adoption_unused_skips=self._adoption_unused_skips, cache_adoption_total_ms=self._adoption_total_ms, cache_adoption_max_ms=self._adoption_max_ms, pinned_free_bytes=sum(size for _, size in self._pinned_free), diff --git a/tests/test_runtime/test_hs_unused_work.py b/tests/test_runtime/test_hs_unused_work.py new file mode 100644 index 0000000000..454b67c61f --- /dev/null +++ b/tests/test_runtime/test_hs_unused_work.py @@ -0,0 +1,124 @@ +"""No-op cache work must disappear without weakening content or ownership checks.""" + +import hashlib +import struct +import unittest +from dataclasses import replace +from unittest import mock + +import torch + +from specforge_sglang_capture.hs import ( + CaptureContractError, CaptureController, CapturedSpan, ExecutionIdentity, + FeatureSchema, FeatureSpec, PrefixFeatureCache, +) + + +def schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1,)),)) + + +class UnusedCacheWorkTests(unittest.TestCase): + def cache(self, capacity=1024, enabled=True): + with mock.patch.dict("os.environ", {"SPECFORGE_HS_SKIP_UNUSED_CACHE_WORK": str(int(enabled))}): + return PrefixFeatureCache(schema(), 4, capacity) + + def test_empty_lease_skips_hash_but_validates_all_tokens_and_identity(self): + cache = self.cache() + tokens = tuple(range(8)) + lease = cache.lookup_and_pin("ns", tokens, 0) + with mock.patch.object(cache, "block_keys", side_effect=AssertionError("unnecessary hash")): + cache._validate_plan_lease(lease, 0, "ns", tokens) + for bad in [(True,), (-1,), (2**63,), [0]]: + with self.assertRaises(CaptureContractError): + cache._validate_plan_lease(lease, 0, "ns", bad) + for handle, end, namespace in [(replace(lease), 0, "ns"), (lease, 1, "ns"), + (lease, 0, "other"), (lease, -1, "ns")]: + with self.assertRaises(CaptureContractError): + cache._validate_plan_lease(handle, end, namespace, tokens) + cache.release_lease(lease) + with self.assertRaises(CaptureContractError): + cache._validate_plan_lease(lease, 0, "ns", tokens) + + def test_disabled_cache_lookup_validates_namespace_and_tokens_without_content_hash(self): + cache = self.cache(0) + with mock.patch.object(cache, "block_keys", side_effect=AssertionError("unnecessary hash")): + lease = cache.lookup_and_pin("ns", tuple(range(8)), 8) + self.assertEqual(lease.coverage_end, 0) + self.assertEqual(cache.stats()["active_leases"], 1) + cache.release_lease(lease) + for namespace, tokens, limit in [("", (0,), 1), (None, (0,), 1), + ("ns", (True,), 1), ("ns", (0,), -1)]: + with self.assertRaises(CaptureContractError): + cache.lookup_and_pin(namespace, tokens, limit) + + def test_protocol_bytes_and_parent_chain_match_independent_v1_oracle(self): + tokens = (0, 1, 2**31, 2**63-1, 9, 8, 7, 6, 5) + namespace = "model-测试" + encoded = namespace.encode() + parent = hashlib.sha256(b"spec-prefix-root-v1\0" + struct.pack(" Date: Tue, 10 Nov 2026 18:01:04 +0800 Subject: [PATCH 13/17] feat(hs): add bounded opt-in publication boundary diagnostics --- .../specforge_sglang_capture/hs/runtime.py | 56 +++++++++ .../specforge_sglang_capture/sink.py | 45 +++++++- .../test_hs_pipeline_diagnostics.py | 106 ++++++++++++++++++ 3 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 tests/test_runtime/test_hs_pipeline_diagnostics.py diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py index ed2b794fa5..b3e4168d0c 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py @@ -63,6 +63,7 @@ class _Publication: cleanup: Future | None = None failed: BaseException | None = None publish_started: float = 0.0 + diagnostic: dict | None = None class HSCompletion: @@ -90,6 +91,14 @@ def result(self): result = self.job.output.result() except Exception as exc: result = {"sample_id": self.job.spec["sample_id"], "error": str(exc)} + if self.job.diagnostic is not None: + now = time.monotonic_ns() + self.job.diagnostic["p0_v1_response_callback_ns"] = now + observed = self.job.diagnostic.get("p0_v1_owner_observed_ns") + if observed is not None: + elapsed = (now - observed) / 1e6 + self.job.diagnostic["p0_v1_owner_to_response_callback_ms"] = elapsed + result.setdefault("timings", {})["p0_v1_owner_to_response_callback_ms"] = elapsed self.runtime.capture.results[self.job.execution.request_id] = result self._resolved = True @@ -157,6 +166,12 @@ def create_core(): self.candidates: dict[str, _Candidate] = {} self.jobs: dict[ExecutionIdentity, _Publication] = {} self.history = deque(maxlen=256) + self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" + self._diagnostic_every = max(1, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_EVERY", "1"))) + self._diagnostic_limit = max(0, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_MAX", "4096"))) + self._diagnostic_sequence = 0 + self._diagnostic_batch = 0 + self.diagnostics = deque(maxlen=self._diagnostic_limit) self._transferred_tokens = [] self._closed = False self.fatal_capture_error = None @@ -590,6 +605,16 @@ def snapshot(self, req, item, aux, last_hidden): from specforge_sglang_capture.capture import request_spec job = _Publication(plan.execution, candidate.entry, copy.deepcopy(request_spec(req)), plan, future, output) + if self._diagnostics: + sequence = self._diagnostic_sequence + self._diagnostic_sequence += 1 + if sequence % self._diagnostic_every == 0 and sequence // self._diagnostic_every < self._diagnostic_limit: + job.diagnostic = {"p0_v1_event": "publication", "server_epoch": self.epoch, + "request_id": plan.execution.request_id, "attempt_id": plan.execution.attempt_id, + "execution_epoch": plan.execution.execution_epoch, "rank": self.tp_rank, + "store_id": job.spec["store_id"], "sample_id": job.spec["sample_id"], + "request_digest": job.entry.request_digest, "N": plan.N, "L": plan.L, + "suffix_tokens": plan.N - plan.L, "p0_v1_snapshot_ns": time.monotonic_ns()} self.jobs[plan.execution] = job candidate.entry.deliveries += 1 return job @@ -597,7 +622,15 @@ def snapshot(self, req, item, aux, last_hidden): def consume(self, jobs, batch): self._assert_owner() reqs = {req.rid: req for req in batch.reqs} + if self._diagnostics: + self._diagnostic_batch += 1 for job in jobs: + if job.diagnostic is not None: + # One ID per native consume batch; do not sum shared batch work + # once per sample. This timestamp is NOT a CUDA forward timer. + job.diagnostic.update(p0_v1_consume_batch_id=self._diagnostic_batch, + p0_v1_consume_batch_samples=len(jobs), + p0_v1_consume_ns=time.monotonic_ns()) req = reqs.get(job.execution.request_id) if req is None or not self._current(job): self._cancel_job(job, "capture request disappeared before consume") @@ -701,6 +734,7 @@ def poll(self, scheduler=None): if ready: samples = [(job.spec, job.sample.features["aux"], job.sample.features["last_hidden"]) for job in ready] + enqueue_ns = time.monotonic_ns() if self._diagnostics else 0 try: future = self.capture.sink.submit_samples(samples) except Exception as exc: @@ -710,12 +744,18 @@ def poll(self, scheduler=None): for index, job in enumerate(ready): job.sink, job.sink_index, job.sink_count = future, index, len(ready) job.publish_started = time.perf_counter() + if job.diagnostic is not None: + job.diagnostic["p0_v1_runtime_sink_enqueue_ns"] = enqueue_ns + done_ns = job.sample.timings.get("p0_v1_materialize_done_ns") + if done_ns is not None: + job.diagnostic["p0_v1_materialize_to_enqueue_ms"] = (enqueue_ns - done_ns) / 1e6 job.entry.state = "PUBLISHING" samples = None # The sink now owns and clears its argument list. ready.clear() for job in tuple(self.jobs.values()): if job.sink is None or not job.sink.done() or job.cleanup is not None: continue + observed_ns = time.monotonic_ns() if job.diagnostic is not None else 0 try: results = job.sink.result() if len(results) != job.sink_count: @@ -733,6 +773,18 @@ def poll(self, scheduler=None): raise CaptureContractError("capture transfer precedes all execution drains") timings = dict(job.sample.timings) timings["publication_queue_ms"] = (time.perf_counter() - job.publish_started) * 1000 + if job.diagnostic is not None: + observed = observed_ns + sink_timing = dict(getattr(job.sink, "p0_v1_timings", {})) + job.diagnostic.update(timings) + job.diagnostic.update(sink_timing) + job.diagnostic["p0_v1_owner_observed_ns"] = observed + if "p0_v1_sink_done_ns" in sink_timing: + job.diagnostic["p0_v1_sink_done_to_owner_ms"] = (observed - sink_timing["p0_v1_sink_done_ns"]) / 1e6 + job.diagnostic["p0_v1_sink_queue_ms"] = (sink_timing["p0_v1_sink_start_ns"] - sink_timing["p0_v1_sink_enqueue_ns"]) / 1e6 + timings.update({key: value for key, value in job.diagnostic.items() + if key.startswith("p0_v1_")}) + self.diagnostics.append(job.diagnostic) entry.last_plan["timings"] = timings # Sink has cleared its source refs. Transfer allocation ownership # to cache (or retire it), then issue the physical-key certificate. @@ -861,6 +913,10 @@ def stats(): "pending_deliveries": len(self.jobs), "active_readers": sum(len(x.readers) for x in self.registry.active.values()), "plans": list(self.history), + **({"p0_v1_diagnostics": list(self.diagnostics), + "p0_v1_diagnostic_seen": self._diagnostic_sequence, + "p0_v1_diagnostic_every": self._diagnostic_every, + "p0_v1_diagnostic_limit": self._diagnostic_limit} if self._diagnostics else {}), "cache": self.cache.stats() if self.cache else {}, "controller": self.controller.stats() if self.controller else {}} return self._local_phase("control.stats", stats) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py index d211e9fabb..9c77ab0b05 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py @@ -330,6 +330,11 @@ def __init__(self, aux_layer_ids: Optional[Sequence[int]] = None) -> None: self._uncertain_publications = [] self._closed = False self.require_hard_pin = False + self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" + self._diagnostic_every = max(1, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_EVERY", "1"))) + self._diagnostic_limit = max(0, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_MAX", "4096"))) + self._diagnostic_sequence = 0 + self._diagnostic_local = threading.local() # One writer is enough: Mooncake stripes a batched transfer itself. # The thread only decouples that transfer from the scheduler. self._executor = ThreadPoolExecutor( @@ -513,9 +518,26 @@ def submit_samples( raise RuntimeError("capture sink is closed") # Own a separate list: the caller may still be assigning completions # while this worker finishes and drops its last source references. - return self._executor.submit(self._put_ready_samples, list(samples), ready_event) + diagnostic = None + if self._diagnostics: + with self._lock: + sequence = self._diagnostic_sequence + self._diagnostic_sequence += 1 + if sequence % self._diagnostic_every == 0 and sequence // self._diagnostic_every < self._diagnostic_limit: + diagnostic = {"p0_v1_sink_batch_id": sequence, + "p0_v1_sink_batch_samples": len(samples), + "p0_v1_sink_enqueue_ns": time.monotonic_ns()} + future = self._executor.submit(self._put_ready_samples, list(samples), ready_event, diagnostic) + # Scalar-only metadata: never retain another tensor/owner reference. + # The worker finishes writing it before the Future becomes done. + if diagnostic is not None: + future.p0_v1_timings = diagnostic + return future - def _put_ready_samples(self, samples, ready_event): + def _put_ready_samples(self, samples, ready_event, diagnostic=None): + if diagnostic is not None: + self._diagnostic_local.current = diagnostic + diagnostic["p0_v1_sink_start_ns"] = time.monotonic_ns() error = None try: if ready_event is not None: @@ -528,6 +550,9 @@ def _put_ready_samples(self, samples, ready_event): error = scalar_error(exc) finally: samples.clear() + if diagnostic is not None: + diagnostic["p0_v1_sink_done_ns"] = time.monotonic_ns() + self._diagnostic_local.current = None if error is not None: raise error return result @@ -653,9 +678,14 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): ) self._remove_many_quiet(replace_keys) registered: List[int] = [] + registered_bytes = 0 seen_storage = set() submitted = quarantined = False register_started = time.perf_counter() + diagnostic = getattr(self._diagnostic_local, "current", None) + if diagnostic is not None: + diagnostic.update(p0_v1_sink_objects=len(keys), p0_v1_sink_bytes=sum(sizes), + p0_v1_sink_staging_ms=materialize_ms) try: for tensor, nbytes in zip(tensors, sizes): storage = tensor.untyped_storage() @@ -672,6 +702,8 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): else: if rc is None or int(rc) == 0: registered.append(ptr) + if diagnostic is not None: + registered_bytes += storage.nbytes() elif tensor.is_cuda: raise RuntimeError( f"GPU capture buffer registration failed ({rc}); " @@ -679,6 +711,9 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): "PyTorch expandable_segments for this server" ) register_ms = (time.perf_counter() - register_started) * 1000.0 + if diagnostic is not None: + diagnostic["p0_v1_sink_register_ms"] = register_ms + diagnostic["p0_v1_sink_registered_bytes"] = registered_bytes put_started = time.perf_counter() batch_put = getattr(store, "batch_put_from", None) submitted = True @@ -695,6 +730,8 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): self._put_config, ) put_ms = (time.perf_counter() - put_started) * 1000.0 + if diagnostic is not None: + diagnostic["p0_v1_sink_put_ms"] = put_ms if statuses is None: statuses = [0] * len(keys) if len(statuses) != len(keys): @@ -717,12 +754,16 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): self._remove_many_quiet(keys) raise finally: + unregister_started = time.monotonic_ns() if diagnostic is not None else 0 if not quarantined: for ptr in registered: try: store.unregister_buffer(ptr) except Exception: pass + if diagnostic is not None: + diagnostic["p0_v1_sink_unregister_ms"] = (time.monotonic_ns() - unregister_started) / 1e6 + diagnostic["p0_v1_sink_quarantined"] = quarantined if timing_enabled: logger.info( diff --git a/tests/test_runtime/test_hs_pipeline_diagnostics.py b/tests/test_runtime/test_hs_pipeline_diagnostics.py new file mode 100644 index 0000000000..5df18e496c --- /dev/null +++ b/tests/test_runtime/test_hs_pipeline_diagnostics.py @@ -0,0 +1,106 @@ +"""CPU timing-boundary tests; these do not claim CUDA/RDMA performance.""" +import os +import threading +import unittest +from unittest import mock + +import torch + +from tests.test_runtime import test_hs_plugin as fixture + +_Store = fixture._Store +from specforge_sglang_capture.sink import SpecCaptureSink + + +class PipelineDiagnosticsTests(unittest.TestCase): + setUp = fixture.HSPluginTests.setUp + tearDown = fixture.HSPluginTests.tearDown + runtime = fixture.HSPluginTests.runtime + request = fixture.HSPluginTests.request + admit = fixture.HSPluginTests.admit + forward = fixture.HSPluginTests.forward + spin = fixture.HSPluginTests.spin + + def test_default_off_preserves_response_and_does_not_record(self): + with mock.patch.dict(os.environ, {"SPECFORGE_HS_DIAGNOSTICS": "0"}): + hs = self.runtime() + req = self.request(hs) + self.admit(hs, req) + output = self.forward(hs, [req], [8]) + self.spin(hs, lambda: hs.deferred[0][1].done()) + hs.deferred[0][1].result() + self.assertIsNone(output.hs_jobs[0].diagnostic) + self.assertFalse(hs.diagnostics) + result = hs.capture.results[req.rid] + self.assertFalse(any(k.startswith("p0_v1_") for k in result["timings"])) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + + def test_sampled_boundaries_preserve_writer_drain_and_bounded_records(self): + gate = threading.Event() + store = _Store(gate=gate) + with mock.patch.dict(os.environ, {"SPECFORGE_HS_DIAGNOSTICS": "1", + "SPECFORGE_HS_DIAGNOSTICS_EVERY": "1", "SPECFORGE_HS_DIAGNOSTICS_MAX": "1"}): + hs = self.runtime(store=store) + reqs = [self.request(hs, name) for name in ("a", "b")] + for req in reqs: + self.admit(hs, req) + output = self.forward(hs, reqs, [8, 8], wait_materialize=True) + first, second = output.hs_jobs + self.assertIs(first.sink, second.sink) + self.assertTrue(store.entered.wait(2)) + try: + self.assertFalse(first.sink.done()) + self.assertFalse(first.output.done()) + self.assertGreater(hs.controller.stats()["output_bytes"], 0) + self.assertTrue(store.registered) + self.assertNotIn("p0_v1_sink_done_ns", first.sink.p0_v1_timings) + finally: + gate.set() + first.sink.result(timeout=3) + self.assertFalse(store.registered) + self.assertFalse(first.output.done()) # Native owner must still observe. + hs.poll() + for _, completion in hs.deferred: + completion.result() + diag, = hs.diagnostics + self.assertIs(diag, first.diagnostic) + self.assertIsNone(second.diagnostic) + keys = ["p0_v1_sink_enqueue_ns", "p0_v1_sink_start_ns", "p0_v1_sink_done_ns", + "p0_v1_owner_observed_ns", "p0_v1_response_callback_ns"] + self.assertEqual([diag[k] for k in keys], sorted(diag[k] for k in keys)) + self.assertLessEqual(diag["p0_v1_materialize_done_ns"], diag["p0_v1_runtime_sink_enqueue_ns"]) + self.assertGreaterEqual(diag["p0_v1_materialize_to_enqueue_ms"], 0) + self.assertEqual(diag["p0_v1_sink_batch_samples"], 2) + self.assertEqual(diag["p0_v1_consume_batch_samples"], 2) + for name in ("register", "put", "unregister", "queue"): + self.assertGreaterEqual(diag["p0_v1_sink_" + name + "_ms"], 0) + self.assertGreater(diag["p0_v1_sink_registered_bytes"], 0) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + + def test_sink_queue_is_before_writer_start_and_metadata_has_no_tensors(self): + with mock.patch.dict(os.environ, {"SPECFORGE_HS_DIAGNOSTICS": "1"}): + writer = SpecCaptureSink() + gate, started = threading.Event(), threading.Event() + self.addCleanup(writer.close) + def block(): + started.set() + self.assertTrue(gate.wait(3)) + blocker = writer._executor.submit(block) + self.assertTrue(started.wait(2)) + writer._store = _Store() + spec = {"store_id": "test", "sample_id": "a", "features": {"aux": "hidden"}} + future = writer.submit_samples([(spec, torch.ones(2, 2), None)]) + try: + self.assertNotIn("p0_v1_sink_start_ns", future.p0_v1_timings) + finally: + gate.set() + blocker.result(timeout=3) + future.result(timeout=3) + d = future.p0_v1_timings + self.assertLessEqual(d["p0_v1_sink_enqueue_ns"], d["p0_v1_sink_start_ns"]) + self.assertLessEqual(d["p0_v1_sink_start_ns"], d["p0_v1_sink_done_ns"]) + self.assertTrue(all(type(value) in (int, float, bool) for value in d.values())) + + +if __name__ == "__main__": + unittest.main() From 544d097a7d86d376f2be9957bda54579f73b4c43 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 18:01:38 +0800 Subject: [PATCH 14/17] diag(hs): split bounded materialization timings behind opt-in flag --- .../specforge_sglang_capture/hs/controller.py | 58 ++++++++++++++++++- tests/test_runtime/test_hs_unused_work.py | 16 +++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py index 2b4dca0d52..aed3292064 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py @@ -6,11 +6,13 @@ from __future__ import annotations +import os +import resource import threading import uuid from collections import deque from concurrent.futures import Future, ThreadPoolExecutor -from time import perf_counter +from time import monotonic_ns, perf_counter from dataclasses import dataclass, field import torch @@ -60,6 +62,7 @@ class _Job: is_cuda: bool state: _Execution future: Future + diagnostics: dict[str, int | float] = field(default_factory=dict) class _UnsafeDrain(RuntimeError): @@ -79,6 +82,7 @@ def __init__(self, cache: PrefixFeatureCache, schema: FeatureSchema, if not isinstance(cache, PrefixFeatureCache) or schema != cache.schema: raise CaptureContractError("controller/cache schema mismatch") self.cache, self.schema = cache, schema + self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" self._limits = { "pinned": checked_int(pinned_budget_bytes, "pinned_budget_bytes"), "output": checked_int(output_budget_bytes, "output_budget_bytes"), @@ -371,6 +375,8 @@ def submit(self, plan: CapturePlan, span: CapturedSpan) -> Future[MaterializedSa # Future.cancel() cannot cancel the cleanup worker and strand storage. future.set_running_or_notify_cancel() job = _Job(plan, features, span.ready_event, is_cuda, state, future) + if self._diagnostics: + job.diagnostics["p0_v1_materialize_submitted_ns"] = monotonic_ns() try: self._executor.submit(self._run_job, job) except BaseException: @@ -389,7 +395,10 @@ def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: if not job.is_cuda: if job.ready_event is not None: try: + ready_started = monotonic_ns() if self._diagnostics else 0 job.ready_event.synchronize() + if self._diagnostics: + job.diagnostics["p0_v1_producer_ready_wait_ms"] = (monotonic_ns() - ready_started) / 1e6 except BaseException as exc: raise _UnsafeDrain(exc, [job.ready_event]) from exc return {} if self._cancelled(job) else dict(job.features) @@ -411,15 +420,31 @@ def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: raise CaptureContractError("pinned arena quarantined after unproven DMA drain") stream = torch.cuda.Stream(device=device) with torch.cuda.stream(stream): + if self._diagnostics: + waiting = torch.cuda.Event(enable_timing=True) + ready = torch.cuda.Event(enable_timing=True) + copy_start = torch.cuda.Event(enable_timing=True) + waiting.record(stream) stream.wait_event(job.ready_event) + if self._diagnostics: + ready.record(stream) + alloc_started = monotonic_ns() staging = self._staging_views(job.plan.N - job.plan.L, job.plan.reservation) + if self._diagnostics: + job.diagnostics["p0_v1_staging_alloc_ms"] = (monotonic_ns() - alloc_started) / 1e6 + copy_start.record(stream) for name, source in job.features.items(): target = staging[name] target.copy_(source, non_blocking=True) source.record_stream(stream) - copied = torch.cuda.Event() + copied = torch.cuda.Event(enable_timing=True) if self._diagnostics else torch.cuda.Event() copied.record(stream) copied.synchronize() + if self._diagnostics: + # Read events only after the existing real copy drain. No + # timing-only synchronization is introduced on the hot path. + job.diagnostics["p0_v1_producer_ready_wait_ms"] = waiting.elapsed_time(ready) + job.diagnostics["p0_v1_d2h_copy_ms"] = copy_start.elapsed_time(copied) return staging except BaseException as exc: drain_errors = [] @@ -459,6 +484,11 @@ def _run_job(self, job: _Job) -> None: completion = None unsafe = False try: + if self._diagnostics: + started_ns = monotonic_ns() + job.diagnostics["p0_v1_materialize_started_ns"] = started_ns + job.diagnostics["p0_v1_d2h_queue_ms"] = (started_ns - job.diagnostics["p0_v1_materialize_submitted_ns"]) / 1e6 + job.diagnostics["p0_v1_d2h_bytes"] = (job.plan.N - job.plan.L) * self.schema.row_bytes if job.is_cuda else 0 copy_started = perf_counter() staging = self._copy_suffix(job) copied_at = perf_counter() @@ -467,14 +497,32 @@ def _run_job(self, job: _Job) -> None: if self._cancelled(job): raise CaptureCancelledError(job.state.reason or "execution released") with torch.no_grad(): + if self._diagnostics: + alloc_started = monotonic_ns() + faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt for spec in self.schema.features: output[spec.name] = torch.empty((job.plan.N, spec.width), dtype=DTYPES[spec.dtype], device="cpu") + if self._diagnostics: + prefix_started = monotonic_ns() + job.diagnostics["p0_v1_output_alloc_ms"] = (prefix_started - alloc_started) / 1e6 + job.diagnostics["p0_v1_output_alloc_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started + faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt try: self.cache.copy_prefix_into(job.plan.lease, job.plan.L, output) finally: self.cache.release_lease(job.plan.lease) + if self._diagnostics: + suffix_started = monotonic_ns() + job.diagnostics["p0_v1_prefix_copy_ms"] = (suffix_started - prefix_started) / 1e6 + job.diagnostics["p0_v1_prefix_copy_bytes"] = job.plan.L * self.schema.row_bytes + job.diagnostics["p0_v1_prefix_copy_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started + faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt for name in output: output[name][job.plan.L:job.plan.N].copy_(staging[name]) + if self._diagnostics: + job.diagnostics["p0_v1_suffix_copy_ms"] = (monotonic_ns() - suffix_started) / 1e6 + job.diagnostics["p0_v1_suffix_copy_bytes"] = (job.plan.N - job.plan.L) * self.schema.row_bytes + job.diagnostics["p0_v1_suffix_copy_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started staging.clear() with self._lock: self._release_transient_locked(self._reservation(job.plan.reservation)) @@ -484,8 +532,14 @@ def _run_job(self, job: _Job) -> None: assembled_at = perf_counter() cache_keys = self._prepare_cache_adoption(job) filled_at = perf_counter() + if self._diagnostics: + job.diagnostics["p0_v1_key_prepare_ms"] = (filled_at - assembled_at) * 1000.0 + # Assembly and key preparation are complete. Sample/completion + # bookkeeping and future delivery follow on this same worker. + job.diagnostics["p0_v1_materialize_done_ns"] = monotonic_ns() sample = MaterializedSample(job.plan.execution, output, job.plan.N, lambda: self._sample_released(job.plan.reservation), timings={ + **job.diagnostics, "d2h_wait_ms": (copied_at - copy_started) * 1000.0, "assembly_ms": (assembled_at - copied_at) * 1000.0, "cache_fill_ms": (filled_at - assembled_at) * 1000.0, diff --git a/tests/test_runtime/test_hs_unused_work.py b/tests/test_runtime/test_hs_unused_work.py index 454b67c61f..74eaf0026d 100644 --- a/tests/test_runtime/test_hs_unused_work.py +++ b/tests/test_runtime/test_hs_unused_work.py @@ -82,6 +82,7 @@ def capture(self, cache, n=8): sample = controller.submit(plan, span).result(5) self.assertTrue(torch.equal(sample.features["aux"], features["aux"])) self.assertEqual(controller.stats()["output_bytes"], n * schema().row_bytes) + self.last_timings = dict(sample.timings) sample.release() sample.release() controller.poll_completed() @@ -111,6 +112,21 @@ def test_cold_cache_on_still_adopts_and_rejects_wrong_content_lease(self): cache._validate_plan_lease(lease, 4, "ns", (99,) + tuple(range(1, 8))) cache.release_lease(lease) + def test_diagnostics_are_opt_in_and_describe_cpu_copy_without_changing_output(self): + for enabled in (False, True): + with mock.patch.dict("os.environ", {"SPECFORGE_HS_DIAGNOSTICS": str(int(enabled))}): + self.capture(self.cache(0)) + timings = self.last_timings + if enabled: + self.assertLessEqual(timings["p0_v1_materialize_submitted_ns"], timings["p0_v1_materialize_started_ns"]) + self.assertLessEqual(timings["p0_v1_materialize_started_ns"], timings["p0_v1_materialize_done_ns"]) + self.assertEqual(timings["p0_v1_d2h_bytes"], 0) + self.assertEqual(timings["p0_v1_prefix_copy_bytes"], 0) + self.assertEqual(timings["p0_v1_suffix_copy_bytes"], 8 * schema().row_bytes) + self.assertGreaterEqual(timings["p0_v1_suffix_copy_ms"], 0) + else: + self.assertFalse(any(key.startswith("p0_") for key in timings)) + def test_rollback_flag_preserves_legacy_hash_and_adoption_work(self): cache = self.cache(0, enabled=False) with mock.patch.object(cache, "block_keys", wraps=cache.block_keys) as keys: From 2751f90c5574c8c646cbc414bac0e1103a748f35 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 18:03:08 +0800 Subject: [PATCH 15/17] fix(hs): avoid duplicate token scans on validated hash path --- .../sglang-spec-capture/specforge_sglang_capture/hs/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py index 63d300535a..f1938d623a 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py @@ -178,7 +178,7 @@ def _checked_lease(self, lease: PrefixLease, end: int, raise CaptureContractError("lease does not cover the requested prefix") if namespace is not None and namespace != rec.namespace: raise CaptureContractError("lease namespace differs from plan") - if tokens is not None: + if tokens is not None and not end and self.skip_unused_cache_work: _token_tuple(tokens) if tokens is not None and (end or not self.skip_unused_cache_work): expected = self.block_keys(rec.namespace, tokens) From 1fbaac66db5c1d43d73e9f8b255238ef78dbf655 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 18:07:10 +0800 Subject: [PATCH 16/17] fix(diag): bound controller timing events by sample policy --- .../specforge_sglang_capture/hs/controller.py | 34 +++++++++++-------- tests/test_runtime/test_hs_unused_work.py | 22 ++++++++++++ 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py index aed3292064..8a8fb8b16f 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py @@ -83,6 +83,9 @@ def __init__(self, cache: PrefixFeatureCache, schema: FeatureSchema, raise CaptureContractError("controller/cache schema mismatch") self.cache, self.schema = cache, schema self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" + self._diagnostic_every = max(1, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_EVERY", "1"))) + self._diagnostic_limit = max(0, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_MAX", "4096"))) + self._diagnostic_sequence = 0 self._limits = { "pinned": checked_int(pinned_budget_bytes, "pinned_budget_bytes"), "output": checked_int(output_budget_bytes, "output_budget_bytes"), @@ -376,7 +379,10 @@ def submit(self, plan: CapturePlan, span: CapturedSpan) -> Future[MaterializedSa future.set_running_or_notify_cancel() job = _Job(plan, features, span.ready_event, is_cuda, state, future) if self._diagnostics: - job.diagnostics["p0_v1_materialize_submitted_ns"] = monotonic_ns() + sequence = self._diagnostic_sequence + self._diagnostic_sequence += 1 + if sequence % self._diagnostic_every == 0 and sequence // self._diagnostic_every < self._diagnostic_limit: + job.diagnostics["p0_v1_materialize_submitted_ns"] = monotonic_ns() try: self._executor.submit(self._run_job, job) except BaseException: @@ -395,9 +401,9 @@ def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: if not job.is_cuda: if job.ready_event is not None: try: - ready_started = monotonic_ns() if self._diagnostics else 0 + ready_started = monotonic_ns() if job.diagnostics else 0 job.ready_event.synchronize() - if self._diagnostics: + if job.diagnostics: job.diagnostics["p0_v1_producer_ready_wait_ms"] = (monotonic_ns() - ready_started) / 1e6 except BaseException as exc: raise _UnsafeDrain(exc, [job.ready_event]) from exc @@ -420,27 +426,27 @@ def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: raise CaptureContractError("pinned arena quarantined after unproven DMA drain") stream = torch.cuda.Stream(device=device) with torch.cuda.stream(stream): - if self._diagnostics: + if job.diagnostics: waiting = torch.cuda.Event(enable_timing=True) ready = torch.cuda.Event(enable_timing=True) copy_start = torch.cuda.Event(enable_timing=True) waiting.record(stream) stream.wait_event(job.ready_event) - if self._diagnostics: + if job.diagnostics: ready.record(stream) alloc_started = monotonic_ns() staging = self._staging_views(job.plan.N - job.plan.L, job.plan.reservation) - if self._diagnostics: + if job.diagnostics: job.diagnostics["p0_v1_staging_alloc_ms"] = (monotonic_ns() - alloc_started) / 1e6 copy_start.record(stream) for name, source in job.features.items(): target = staging[name] target.copy_(source, non_blocking=True) source.record_stream(stream) - copied = torch.cuda.Event(enable_timing=True) if self._diagnostics else torch.cuda.Event() + copied = torch.cuda.Event(enable_timing=True) if job.diagnostics else torch.cuda.Event() copied.record(stream) copied.synchronize() - if self._diagnostics: + if job.diagnostics: # Read events only after the existing real copy drain. No # timing-only synchronization is introduced on the hot path. job.diagnostics["p0_v1_producer_ready_wait_ms"] = waiting.elapsed_time(ready) @@ -484,7 +490,7 @@ def _run_job(self, job: _Job) -> None: completion = None unsafe = False try: - if self._diagnostics: + if job.diagnostics: started_ns = monotonic_ns() job.diagnostics["p0_v1_materialize_started_ns"] = started_ns job.diagnostics["p0_v1_d2h_queue_ms"] = (started_ns - job.diagnostics["p0_v1_materialize_submitted_ns"]) / 1e6 @@ -497,12 +503,12 @@ def _run_job(self, job: _Job) -> None: if self._cancelled(job): raise CaptureCancelledError(job.state.reason or "execution released") with torch.no_grad(): - if self._diagnostics: + if job.diagnostics: alloc_started = monotonic_ns() faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt for spec in self.schema.features: output[spec.name] = torch.empty((job.plan.N, spec.width), dtype=DTYPES[spec.dtype], device="cpu") - if self._diagnostics: + if job.diagnostics: prefix_started = monotonic_ns() job.diagnostics["p0_v1_output_alloc_ms"] = (prefix_started - alloc_started) / 1e6 job.diagnostics["p0_v1_output_alloc_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started @@ -511,7 +517,7 @@ def _run_job(self, job: _Job) -> None: self.cache.copy_prefix_into(job.plan.lease, job.plan.L, output) finally: self.cache.release_lease(job.plan.lease) - if self._diagnostics: + if job.diagnostics: suffix_started = monotonic_ns() job.diagnostics["p0_v1_prefix_copy_ms"] = (suffix_started - prefix_started) / 1e6 job.diagnostics["p0_v1_prefix_copy_bytes"] = job.plan.L * self.schema.row_bytes @@ -519,7 +525,7 @@ def _run_job(self, job: _Job) -> None: faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt for name in output: output[name][job.plan.L:job.plan.N].copy_(staging[name]) - if self._diagnostics: + if job.diagnostics: job.diagnostics["p0_v1_suffix_copy_ms"] = (monotonic_ns() - suffix_started) / 1e6 job.diagnostics["p0_v1_suffix_copy_bytes"] = (job.plan.N - job.plan.L) * self.schema.row_bytes job.diagnostics["p0_v1_suffix_copy_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started @@ -532,7 +538,7 @@ def _run_job(self, job: _Job) -> None: assembled_at = perf_counter() cache_keys = self._prepare_cache_adoption(job) filled_at = perf_counter() - if self._diagnostics: + if job.diagnostics: job.diagnostics["p0_v1_key_prepare_ms"] = (filled_at - assembled_at) * 1000.0 # Assembly and key preparation are complete. Sample/completion # bookkeeping and future delivery follow on this same worker. diff --git a/tests/test_runtime/test_hs_unused_work.py b/tests/test_runtime/test_hs_unused_work.py index 74eaf0026d..e86bf0afe2 100644 --- a/tests/test_runtime/test_hs_unused_work.py +++ b/tests/test_runtime/test_hs_unused_work.py @@ -127,6 +127,28 @@ def test_diagnostics_are_opt_in_and_describe_cpu_copy_without_changing_output(se else: self.assertFalse(any(key.startswith("p0_") for key in timings)) + def test_diagnostic_sampling_bounds_controller_event_work(self): + with mock.patch.dict("os.environ", {"SPECFORGE_HS_DIAGNOSTICS": "1", + "SPECFORGE_HS_DIAGNOSTICS_EVERY": "2", "SPECFORGE_HS_DIAGNOSTICS_MAX": "2"}): + cache = self.cache(0) + controller = CaptureController(cache, schema(), 512, 512, 512) + sampled = [] + for index in range(6): + execution = ExecutionIdentity("server", str(index), str(index), 0) + tokens = tuple(range(8)) + lease = cache.lookup_and_pin("ns", tokens, 8) + reservation = controller.try_reserve_output(execution, 8, 0, 0) + plan = controller.freeze_plan(execution, "ns", tokens, 8, 0, lease, reservation) + sample = controller.submit(plan, CapturedSpan(execution, 0, 8, + {"aux": torch.zeros(8, 2)}, None)).result(5) + sampled.append("p0_v1_materialize_done_ns" in sample.timings) + sample.release() + controller.poll_completed() + controller.poll_drained() + controller.shutdown() + self.assertEqual(sampled, [True, False, True, False, False, False]) + self.assertEqual(controller.stats()["pending_executions"], 0) + def test_rollback_flag_preserves_legacy_hash_and_adoption_work(self): cache = self.cache(0, enabled=False) with mock.patch.object(cache, "block_keys", wraps=cache.block_keys) as keys: From 391c34d4a1235da6bab3d709b7d0a2d694d690f1 Mon Sep 17 00:00:00 2001 From: Wilbolu Date: Tue, 10 Nov 2026 18:49:48 +0800 Subject: [PATCH 17/17] Retain publication sources when executor acceptance is unknown --- .../specforge_sglang_capture/sink.py | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py index 9c77ab0b05..ec23ca57fe 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py @@ -328,6 +328,7 @@ def __init__(self, aux_layer_ids: Optional[Sequence[int]] = None) -> None: self._control_listener: Optional[socket.socket] = None self._lock = threading.Lock() self._uncertain_publications = [] + self._uncertain_lock = threading.Lock() self._closed = False self.require_hard_pin = False self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" @@ -475,8 +476,16 @@ def report_failure(done): future.add_done_callback(report_failure) return future + def _uncertain_snapshot(self): + with self._uncertain_lock: + return tuple(self._uncertain_publications) + + def _retain_uncertain(self, item): + with self._uncertain_lock: + self._uncertain_publications.append(item) + def _discard_keys(self, keys: List[str]) -> None: - if any(set(keys).intersection(item["keys"]) for item in self._uncertain_publications): + if any(set(keys).intersection(item["keys"]) for item in self._uncertain_snapshot()): raise CaptureWriterUncertain("cannot remove objects before native writer drain") if self._nvlink is not None: self._nvlink.free(keys) @@ -527,7 +536,20 @@ def submit_samples( diagnostic = {"p0_v1_sink_batch_id": sequence, "p0_v1_sink_batch_samples": len(samples), "p0_v1_sink_enqueue_ns": time.monotonic_ns()} - future = self._executor.submit(self._put_ready_samples, list(samples), ready_event, diagnostic) + owned = list(samples) + # submit() may enqueue and then fail while starting a worker. Keep a + # separate immutable owner: the accepted worker clears its own list. + retained = tuple(owned) + keys = tuple(sample_object_key(spec, str(name)) + for spec, _, _ in retained + for name in [*(spec.get("features") or {}).values(), + *(item["name"] for item in spec.get("passthrough") or [])]) + try: + future = self._executor.submit(self._put_ready_samples, owned, ready_event, diagnostic) + except BaseException as exc: + self._retain_uncertain({"samples": retained, "keys": keys, + "registered": (), "acceptance_unknown": True}) + raise CaptureWriterUncertain("capture enqueue acceptance unknown: " + str(exc)) from None # Scalar-only metadata: never retain another tensor/owner reference. # The worker finishes writing it before the Future becomes done. if diagnostic is not None: @@ -560,7 +582,7 @@ def _put_ready_samples(self, samples, ready_event, diagnostic=None): def close(self) -> None: self._closed = True self._executor.shutdown(wait=True, cancel_futures=False) - if self._uncertain_publications: + if self._uncertain_snapshot(): raise RuntimeError("capture sink retains native writes whose drain is unproven") if self._control_listener is not None and self._nvlink is None: self._control_listener.close() @@ -577,7 +599,7 @@ def put_samples(self, samples: List[Sample]) -> List[Dict[str, Any]]: """ if not samples: return [] - if self._uncertain_publications: + if self._uncertain_snapshot(): raise CaptureWriterUncertain("capture native writer is quarantined") store = None if nvlink_enabled() else self._connect() @@ -748,7 +770,7 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): if (getattr(exc, "capture_writer_drained", True) is False or (submitted and not isinstance(exc, _DrainedWriteFailure))): quarantined = True - self._uncertain_publications.append({"tensors": tensors, "keys": keys, + self._retain_uncertain({"tensors": tensors, "keys": keys, "registered": tuple(registered), "bytes": sum(sizes)}) raise CaptureWriterUncertain(str(exc)) from None self._remove_many_quiet(keys)