diff --git a/plugins/sglang-spec-capture/README.md b/plugins/sglang-spec-capture/README.md index 449702bc38..851da0f6cb 100644 --- a/plugins/sglang-spec-capture/README.md +++ b/plugins/sglang-spec-capture/README.md @@ -180,3 +180,82 @@ with RDMA NICs. Without it, Mooncake installs its RDMA transport instead. | `SGLANG_SPEC_CAPTURE_ARENA_BYTES` | Arena size in bytes. Required with `MOONCAKE_PROTOCOL=nvlink`. | | `SGLANG_SPEC_CAPTURE_CONTROL_PORT` | Port of the NVLink free endpoint (default: ephemeral). | | `MOONCAKE_*` | Mooncake client settings. | + +## Opt-in HS capture protocol v1 + +This migration additionally needs SGLang's native +`managers.prefix_admission.CAPTURE_ADMISSION_PROTOCOL == 1` companion extension. +It does not apply the removed legacy capture patch. Enable +`SPECFORGE_HS_CAPTURE=1` to negotiate server-issued attempt reservations at +`POST /spec_capture/control`; set `SPECFORGE_HS_CACHE=1` to reuse hidden-state +prefixes. `HS_CAPTURE=1, HS_CACHE=0` keeps the same complete-output, byte-budget +and immutable-attempt protocol while forcing a zero prefix. The ordinary +non-negotiated capture path remains available when both are off. + +| Variable | Default | +|---|---:| +| `SPECFORGE_HS_CHUNK_TOKENS` | 256 | +| `SPECFORGE_HS_CACHE_BYTES` | 1073741824 | +| `SPECFORGE_HS_PINNED_BYTES` | 268435456 | +| `SPECFORGE_HS_OUTPUT_BYTES` | 536870912 | +| `SPECFORGE_HS_DEVICE_BYTES` | 268435456 | +| `SPECFORGE_HS_MAX_WORKERS` | 4 | +| `SPECFORGE_HS_ATTEMPT_CAPACITY` | 4096 | + +All limits must be positive. Pinned staging uses one retained arena, conservatively +rounded down to a power of two, with aligned independent reservation slots. +Cache adoption charges complete underlying storage, including unmatchable tails. +Output, pinned and device limits are separate. Hard oversize fails explicitly; +temporary admission pressure permits only the native transaction's single +fully-rolled-back reclaim/retry. Atomic registry-capacity rejection returns +`error_code=capture_capacity_declined, reservation_granted=false` before granting +any attempt in that begin operation. + +HS capture supports complete causal text, nonchunked prefill, PP1/DP1, no +offload/session/LoRA/weight updates, and host publication with Mooncake hard pin. +Set `SGLANG_SPEC_CAPTURE_GPU_PUT=0` when using RDMA host staging. Reuse additionally +requires deterministic batch-invariant inference, enabled radix, and an exact +selected recurrent checkpoint backend. Unsupported modes fail startup rather +than changing capture semantics. Capability metadata advertises these limits. + +The content namespace binds model/config/load epoch, ordered raw auxiliary +layers, post-norm last hidden state, dtype/layout, causal position/attention, +TP/KV/checkpoint/backend and deterministic semantics. Tokens form complete parent +hash chains; sample/attempt IDs and loss masks do not enter content keys. +Passthrough data remains part of the registered request digest. + +Each external attempt has immutable keys +`store/sample/g1/attempt/server_epoch/reservation_token/feature`. The native +observer retains independently owned suffix snapshots; its auxiliary output +carries controller jobs and performs no second D2H. Assembly reconstructs every +row `[0,N)` from the leased prefix and the current `[L,N)` suffix. Scheduler safe +points batch ready samples into the native sink, then resolve native deferred +responses. A PUBLISHED certificate is issued only after exact key/schema/digest +validation and source-reader drain. Sink completion permits cache adoption; it +does not retire storage retained by cache leases. Unknown DMA/native transfer +drain quarantines sources, registrations and charges until an external drain +proof exists. Timeouts do not authorize reuse. + +`tests.test_runtime.test_hs_runtime` covers the migrated CPU core; +`tests.test_runtime.test_hs_plugin` covers native-output wiring with a CPU buffer +store; `test_hs_controller_parallel_cuda` requires actual CUDA/pinned copies. +CPU fixtures do not certify numerical model, real Mooncake, scheduler TP or +training equivalence. + +Internal TP control/admission collectives use named local phases: owner/poll, +writer payload construction, received-state application, target lookup and +permit validation all vote local errors before entering a later collective. +Ranks agree request membership and control branches before dispatch. Startup +epoch/model/schema/core failures use the same protocol. `poll` and `lifecycle` +themselves remain local because native completion paths may call them only on +the writer. Begin operations preflight a separate registry on every rank; +only unanimous capacity refusal of uncommitted trials can emit the typed +`reservation_granted=false` proof. A rank disagreement or partial commit fails +closed without that proof. + +`tests.test_runtime.test_hs_tp_protocol` drives the real runtime and companion +SG `PrefixAdmissionCoordinator.phase` with two CPU scheduler threads and a +blocking operation-order rendezvous. Set `SPECFORGE_HS_TEST_SG_SOURCE` to the SG +checkout when the installed editable still points at an upstream baseline. +This detects mismatched collective sequences in the tested faults; actual +distributed TP, CUDA and transport failure validation remains separate. diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py b/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py index 19927b8c99..070ee2fa6c 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/__init__.py @@ -44,6 +44,14 @@ def register() -> None: HookRegistry.register( f"{_SCHEDULER}.get_internal_state", _capture_internal_state, HookType.AFTER ) + from specforge_sglang_capture.hs.config import HSConfig + if HSConfig.from_env().capture: + from specforge_sglang_capture.hs.http import install_control_route + HookRegistry.register( + "sglang.srt.entrypoints.http_server.set_global_state", + install_control_route, + HookType.AFTER, + ) def _capture_streamer_class(streamer_class, scheduler, *args, **kwargs): @@ -69,6 +77,10 @@ def _capture_internal_state(result, scheduler, *args, **kwargs): result.internal_state["spec_capture_backend"] = "plugin" result.internal_state["spec_capture_method"] = runtime.capture_method result.internal_state["spec_capture_aux_layer_ids"] = runtime.sink.aux_layer_ids + hs = getattr(runtime, "hs", None) + if hs is not None: + result.internal_state["spec_capture_hs_protocol"] = 1 + result.internal_state["spec_capture_hs_cache"] = hs.enabled if runtime.is_writer: control = runtime.sink.control_endpoint() if control is not None: diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py b/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py index e6be261553..4e30e7d337 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/capture.py @@ -117,12 +117,19 @@ def collect_capture_items( if spec is not None: error = spec.get("_parse_error") prompt_len = len(req.origin_input_ids) - if error is None and prefix_len: + plan = getattr(req, "_specforge_hs_plan", None) + if error is None and plan is not None: + if (prefix_len != plan.L or length != plan.N - plan.L + or prompt_len != plan.N or plan.execution.request_id != req.rid): + error = "the native capture span differs from its committed HS plan" + elif error is None and "protocol_version" in spec: + error = "HS capture has no committed native admission plan" + elif error is None and prefix_len: error = ( f"{prefix_len} prompt tokens were served from the prefix " "cache; spec capture needs a full prefill" ) - elif error is None and length != prompt_len: + elif error is None and plan is None and length != prompt_len: error = ( f"the prefill covered {length} of {prompt_len} prompt tokens; " "launch the server with --chunked-prefill-size -1" @@ -185,9 +192,25 @@ def after_forward(self, batch, forward_batch, logits_output, *, can_run_graph): ), ) checked.append(item) - return CaptureDeviceOutput.from_rows( - self.runtime, checked, aux, last_hidden, copy=can_run_graph - ) + jobs, native_items = [], [] + reqs = {req.rid: req for req in batch.reqs} + for item in checked: + if self.runtime.hs is None or "protocol_version" not in item.spec: + native_items.append(item) + continue + req = reqs[item.rid] + if item.error is None: + try: + jobs.append(self.runtime.hs.snapshot(req, item, aux, last_hidden)) + continue + except Exception as exc: + item = dataclasses.replace(item, error=str(exc)) + self.runtime.hs.capture_error(req, item.error) + native_items.append(item) + output = CaptureDeviceOutput.from_rows( + self.runtime, native_items, aux, last_hidden, copy=can_run_graph) + output.hs_jobs = jobs + return output def _select_rows( @@ -235,6 +258,7 @@ def __init__(self, runtime, items, aux, last_hidden): self.items = items self.aux = aux self.last_hidden = last_hidden + self.hs_jobs = [] @classmethod def from_rows(cls, runtime, items, aux, last_hidden, *, copy): @@ -242,30 +266,36 @@ def from_rows(cls, runtime, items, aux, last_hidden, *, copy): return cls(runtime, items, aux, last_hidden) def copy_to_host(self, copy_tensor): + if not self.items: + # HS controller owns the only D2H path and its bounded pinned arena. + return CaptureHostOutput(self.runtime, [], None, None, hs_jobs=self.hs_jobs) if self.runtime.gpu_put: # Device publication: the sink waits for everything enqueued so far. ready = torch.cuda.Event() ready.record() return CaptureHostOutput( - self.runtime, self.items, self.aux, self.last_hidden, ready_event=ready + self.runtime, self.items, self.aux, self.last_hidden, ready_event=ready, + hs_jobs=self.hs_jobs, ) return CaptureHostOutput( self.runtime, self.items, copy_tensor(self.aux) if self.aux is not None else None, copy_tensor(self.last_hidden) if self.last_hidden is not None else None, + hs_jobs=self.hs_jobs, ) class CaptureHostOutput: """HostAuxiliaryOutput: publish finished capture requests, then hold them.""" - def __init__(self, runtime, items, aux, last_hidden, *, ready_event=None): + def __init__(self, runtime, items, aux, last_hidden, *, ready_event=None, hs_jobs=()): self.runtime = runtime self.items = items self.aux = aux self.last_hidden = last_hidden self.ready_event = ready_event + self.hs_jobs = hs_jobs def _rows(self, tensor, item): if tensor is None: @@ -273,6 +303,9 @@ def _rows(self, tensor, item): return tensor[item.start : item.start + item.length] def consume(self, batch, commits) -> None: + if self.hs_jobs: + self.runtime.hs.consume(self.hs_jobs, batch) + self.hs_jobs = [] reqs = {req.rid: req for req in batch.reqs} samples: List[Sample] = [] held = [] @@ -433,6 +466,7 @@ def __init__( self.results: Dict[str, Dict[str, Any]] = {} self.pending = PendingCaptures(self, max_pending_batches) self.observer = CaptureObserver(self) + self.hs = None def install(scheduler) -> CaptureRuntime: @@ -485,6 +519,18 @@ def install(scheduler) -> CaptureRuntime: ) method = features.aux_hidden_state_capture runtime.capture_method = str(getattr(method, "name", method)).lower() + from specforge_sglang_capture.hs.config import HSConfig + hs_config = HSConfig.from_env() + if hs_config.capture: + from sglang.srt.managers.prefix_admission import CAPTURE_ADMISSION_PROTOCOL + from specforge_sglang_capture.hs.runtime import HSCaptureRuntime + + if CAPTURE_ADMISSION_PROTOCOL != 1: + raise RuntimeError("HS capture requires native admission protocol v1") + if getattr(scheduler, "prefix_admission_observer", None) is not None: + raise RuntimeError("another native prefix admission observer is already installed") + runtime.hs = HSCaptureRuntime(scheduler, runtime, hs_config) + scheduler.prefix_admission_observer = runtime.hs model_runner.forward_observer = runtime.observer scheduler.output_streamer.spec_capture_results = runtime.results scheduler._specforge_capture_runtime = runtime diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py new file mode 100644 index 0000000000..2d64e5643e --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/__init__.py @@ -0,0 +1,45 @@ +"""Single-server content-keyed raw feature cache, frozen Plan A v1 API. + +block_keys(namespace, tokens) returns complete canonical chunk identities. +stats() returns bounded storage/reservation and lifecycle counters. +""" + +from .cache import PrefixFeatureCache +from .controller import CaptureController +from .protocol import ( + AdmissionBudgetError, + CaptureCancelledError, + CaptureCompletion, + CaptureContractError, + CapturePlan, + CapturedSpan, + ContentBlockKey, + DrainHandle, + ExecutionIdentity, + FeatureSchema, + FeatureSpec, + FillReservation, + MaterializedSample, + OutputReservation, + PrefixLease, +) + +__all__ = [ + "AdmissionBudgetError", + "CaptureCancelledError", + "CaptureCompletion", + "CaptureContractError", + "CaptureController", + "CapturePlan", + "CapturedSpan", + "ContentBlockKey", + "DrainHandle", + "ExecutionIdentity", + "FeatureSchema", + "FeatureSpec", + "FillReservation", + "MaterializedSample", + "OutputReservation", + "PrefixFeatureCache", + "PrefixLease", +] diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py new file mode 100644 index 0000000000..e760d4d7e6 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/attempts.py @@ -0,0 +1,137 @@ +"""Scheduler-owned immutable attempt registry, sourced from frozen HS v1.""" +from __future__ import annotations + +import hashlib +import json +import uuid +from collections import deque +from dataclasses import dataclass, field +from typing import Any + + +class CaptureControlRejected(ValueError): + # Structured marker also survives injected in-process control transports. + capture_control_rejected = True + + +class AttemptCapacityDeclined(CaptureControlRejected): + """An atomic begin operation has not granted any reservations.""" + + +def scalar_error(exc): + """Detach completed worker errors from frames that may retain tensor storage.""" + message = str(exc) + exc.__traceback__ = exc.__cause__ = exc.__context__ = None + exc.args = (message,) + error = RuntimeError(message) + if getattr(exc, "capture_writer_drained", True) is False: + error.capture_writer_drained = False + return error + + +def digest_request(tokens, spec): + value = {"tokens": list(tokens), "features": spec.get("features", {}), + "passthrough": spec.get("passthrough", [])} + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +@dataclass +class Attempt: + identity: tuple + attempt_id: str + request_digest: str + token: str + sequence: int + state: str = "RESERVED" + revoked: bool = False + claimed: bool = False + request: Any = None + execution_epoch: int = 0 + drains: list = field(default_factory=list) + deliveries: int = 0 + last_plan: dict = field(default_factory=dict) + object_key_layout: str | None = None + transfer_certificate: dict = field(default_factory=dict) + cleanup_owner: str | None = None + cleanup_token: str | None = None + readers: dict = field(default_factory=dict) + released_readers: deque = field(default_factory=deque) + + +class AttemptRegistry: + """Bounded active reservations. Unknown tokens never authorize generation.""" + def __init__(self, epoch, capacity=4096): + if capacity < 1: + raise ValueError("attempt capacity must be positive") + self.epoch, self.capacity = epoch, capacity + self.active = {} + self.tokens = {} + self.retired = deque(maxlen=capacity) + self.sequence = 0 + + @staticmethod + def identity(body): + identity = (str(body["store_id"]), str(body["sample_id"]), int(body.get("gen", 1))) + if identity[2] != 1: + raise ValueError("capture protocol requires gen=1") + return identity + + def begin(self, body, token=None): + identity = self.identity(body) + old = self.active.get(identity) + attempt_id, digest = str(body["attempt_id"]), str(body["request_digest"]) + if not attempt_id or not digest: + raise ValueError("attempt id and request digest are required") + if old: + if old.attempt_id != attempt_id or old.request_digest != digest: + raise ValueError("prior sample attempt is not retired") + if old.revoked: + raise ValueError("attempt is revoked; resolve and clean it before retry") + return old + if len(self.active) >= self.capacity: + raise AttemptCapacityDeclined("attempt registry capacity exhausted") + self.sequence += 1 + entry = Attempt(identity, attempt_id, digest, token or uuid.uuid4().hex, self.sequence) + entry.object_key_layout = "attempt-v1" if body.get("publication_ownership") == "attempt-v1" else None + self.active[identity] = entry + self.tokens[entry.token] = entry + return entry + + def begin_batch(self, bodies, tokens): + # Validate the complete batch before changing the live registry. + if len(bodies) != len(tokens) or len({self.identity(body) for body in bodies}) != len(bodies): + raise ValueError("duplicate begin_batch identity or token count mismatch") + trial = AttemptRegistry(self.epoch, self.capacity) + trial.active, trial.tokens, trial.sequence = dict(self.active), dict(self.tokens), self.sequence + for body, token in zip(bodies, tokens): + trial.begin(body, token) + self.active, self.tokens, self.sequence = trial.active, trial.tokens, trial.sequence + return [self.active[self.identity(body)] for body in bodies] + + def lookup(self, body): + token = str(body["reservation_token"]) + entry = self.tokens.get(token) + if entry is None or entry.identity != self.identity(body) or entry.attempt_id != str(body["attempt_id"]): + raise ValueError("unknown, retired or mismatched reservation token") + return entry + + def claim(self, body, tokens): + entry = self.lookup(body) + if entry.revoked or entry.claimed: + raise ValueError("reservation revoked or generation already claimed") + if digest_request(tokens, body) != entry.request_digest: + raise ValueError("generate payload differs from reservation") + entry.claimed, entry.state = True, "QUEUED" + return entry + + def retire(self, entry): + if entry.state not in ("WRITER_DRAINED", "PUBLISHED") or entry.deliveries or entry.readers: + raise ValueError("writer is not drained") + if any(not handle.done() for handle in entry.drains): + raise ValueError("runtime readers are not drained") + entry.revoked, entry.state = True, "RETIRED" + self.tokens.pop(entry.token, None) + self.active.pop(entry.identity, None) + entry.request = None + entry.drains.clear() + self.retired.append(entry) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py new file mode 100644 index 0000000000..f1938d623a --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/cache.py @@ -0,0 +1,489 @@ +"""Immutable content chunks with atomic continuous coverage and read leases.""" + +from __future__ import annotations + +import hashlib +import os +import struct +import threading +import uuid +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Mapping + +import torch + +from .protocol import ( + CaptureContractError, ContentBlockKey, ExecutionIdentity, FeatureSchema, + FillReservation, PrefixLease, checked_int, _normalized_error, +) + + +def _token_tuple(tokens) -> tuple[int, ...]: + if not isinstance(tokens, tuple): + raise CaptureContractError("tokens must be an immutable tuple") + if any(type(t) is not int or not 0 <= t < 2**63 for t in tokens): + raise CaptureContractError("tokens must be nonnegative canonical int64 ids") + return tokens + + +@dataclass +class _StorageOwner: + nbytes: int + features: dict[str, torch.Tensor] + keys: set[ContentBlockKey] = field(default_factory=set) + leases: int = 0 + + +@dataclass +class _Entry: + key: ContentBlockKey + nbytes: int + state: str = "WRITING" + features: dict[str, torch.Tensor] = field(default_factory=dict) + leases: int = 0 + owner: _StorageOwner | None = None + + +@dataclass +class _Lease: + handle: PrefixLease + namespace: str + entries: tuple[_Entry, ...] + readers: int = 0 + released: bool = False + + +@dataclass +class _Fill: + handle: FillReservation + entry: _Entry + writer_active: bool = False + abort_requested: bool = False + + +class PrefixFeatureCache: + def __init__(self, schema: FeatureSchema, chunk_tokens: int, capacity_bytes: int): + if not isinstance(schema, FeatureSchema): + raise CaptureContractError("cache requires a FeatureSchema") + self.schema = schema + self.chunk_tokens = checked_int(chunk_tokens, "chunk_tokens", 1) + self.capacity_bytes = checked_int(capacity_bytes, "capacity_bytes") + # Freeze this policy for the lifetime of a cache/controller pair. Setting + # the flag to 0 retains the old work for paired diagnostics and rollback. + self.skip_unused_cache_work = os.environ.get("SPECFORGE_HS_SKIP_UNUSED_CACHE_WORK", "1") != "0" + self._chunk_bytes = self.chunk_tokens * self.schema.row_bytes + self._lock = threading.RLock() + self._entries: dict[ContentBlockKey, _Entry] = {} + self._lru: OrderedDict[ContentBlockKey, None] = OrderedDict() + self._leases: dict[str, _Lease] = {} + self._fills: dict[str, _Fill] = {} + self._bytes = self._peak_bytes = self._evictions = self._fill_skips = 0 + self._logical_peak_bytes = self._adopted_samples = self._adoption_skips = 0 + self._owned_bytes = self._owned_peak_bytes = self._owned_groups = 0 + + def _root_digest(self, namespace: str) -> str: + if not isinstance(namespace, str) or not namespace: + raise CaptureContractError("namespace must be a nonempty server-validated string") + encoded = namespace.encode("utf-8") + return hashlib.sha256(b"spec-prefix-root-v1\0" + struct.pack(" str: + encoded = namespace.encode("utf-8") + try: + parent_bytes = bytes.fromhex(parent) + except (TypeError, ValueError) as exc: + raise CaptureContractError("invalid parent SHA256") from exc + if len(parent_bytes) != 32 or len(parent) != 64: + raise CaptureContractError("parent must be a complete SHA256") + body = (b"spec-prefix-block-v1\0" + struct.pack(" tuple[ContentBlockKey, ...]: + """Complete chunks only; no persistent lookup-side token table.""" + _token_tuple(tokens) + parent = self._root_digest(namespace) + result = [] + for start in range(0, len(tokens) - self.chunk_tokens + 1, self.chunk_tokens): + end = start + self.chunk_tokens + chunk = tokens[start:end] + digest = self._digest(namespace, parent, start, end, chunk) + result.append(ContentBlockKey(namespace, start, end, digest, parent, chunk)) + parent = digest + return tuple(result) + + def _validate_key(self, key: ContentBlockKey) -> None: + if not isinstance(key, ContentBlockKey): + raise CaptureContractError("fill needs a content block key") + self._root_digest(key.namespace) + checked_int(key.start, "block start") + checked_int(key.end, "block end") + if key.start % self.chunk_tokens or key.end - key.start != self.chunk_tokens: + raise CaptureContractError("only aligned complete chunks can be cached") + _token_tuple(key._tokens) + if len(key._tokens) != self.chunk_tokens: + raise CaptureContractError("block tokens do not match its complete range") + if key.start == 0 and key.parent_digest != self._root_digest(key.namespace): + raise CaptureContractError("first block has an invalid root identity") + if key.digest != self._digest(key.namespace, key.parent_digest, key.start, key.end, key._tokens): + raise CaptureContractError("content key does not match its tokens/range") + + def _complete(self, entry: _Entry) -> bool: + if entry.state != "READY": + return False + try: + self.schema.validate(entry.features, self.chunk_tokens, "cpu") + return True + except CaptureContractError: + return False + + def lookup_and_pin(self, namespace: str, tokens: tuple[int, ...], limit: int) -> PrefixLease: + _token_tuple(tokens) + checked_int(limit, "coverage limit") + if self.skip_unused_cache_work and self.capacity_bytes == 0: + # A disabled cache cannot have content. Still validate the external + # namespace/tokens and issue a real, identity-checked empty lease. + self._root_digest(namespace) + keys = () + else: + keys = self.block_keys(namespace, tokens[: min(len(tokens), limit)]) + with self._lock: + entries = [] + for key in keys: + entry = self._entries.get(key) + if entry is None or not self._complete(entry): + break + entries.append(entry) + # The query and every refcount update are one metadata transaction. + for entry in entries: + entry.leases += 1 + if entry.owner is not None: + entry.owner.leases += 1 + self._lru.move_to_end(entry.key) + end = len(entries) * self.chunk_tokens + handle = PrefixLease(uuid.uuid4().hex, end) + self._leases[handle.lease_id] = _Lease(handle, namespace, tuple(entries)) + return handle + + def _checked_lease(self, lease: PrefixLease, end: int, + namespace: str | None = None, tokens: tuple[int, ...] | None = None) -> _Lease: + checked_int(end, "lease read end") + rec = self._leases.get(getattr(lease, "lease_id", None)) + if rec is None or rec.handle is not lease or rec.released: + raise CaptureContractError("lease is stale, foreign, or released") + if end > lease.coverage_end: + raise CaptureContractError("lease does not cover the requested prefix") + if namespace is not None and namespace != rec.namespace: + raise CaptureContractError("lease namespace differs from plan") + if tokens is not None and not end and self.skip_unused_cache_work: + _token_tuple(tokens) + if tokens is not None and (end or not self.skip_unused_cache_work): + expected = self.block_keys(rec.namespace, tokens) + needed = (end + self.chunk_tokens - 1) // self.chunk_tokens + if tuple(e.key for e in rec.entries[:needed]) != expected[:needed]: + raise CaptureContractError("lease content differs from plan") + return rec + + def _validate_plan_lease(self, lease, end, namespace, tokens) -> None: + with self._lock: + self._checked_lease(lease, end, namespace, tokens) + + def shrink_lease(self, lease: PrefixLease, end: int) -> PrefixLease: + with self._lock: + rec = self._checked_lease(lease, end) + if rec.readers: + raise CaptureContractError("cannot shrink a lease while a reader is active") + keep = (end + self.chunk_tokens - 1) // self.chunk_tokens + for entry in rec.entries[keep:]: + entry.leases -= 1 + if entry.owner is not None: + entry.owner.leases -= 1 + self._leases.pop(lease.lease_id) + handle = PrefixLease(uuid.uuid4().hex, end) + self._leases[handle.lease_id] = _Lease(handle, rec.namespace, rec.entries[:keep]) + return handle + + def _finish_release(self, rec: _Lease) -> None: + for entry in rec.entries: + entry.leases -= 1 + if entry.owner is not None: + entry.owner.leases -= 1 + assert entry.owner.leases >= 0 + assert entry.leases >= 0 + self._leases.pop(rec.handle.lease_id, None) + + def release_lease(self, lease: PrefixLease) -> None: + with self._lock: + rec = self._leases.get(getattr(lease, "lease_id", None)) + if rec is None or rec.handle is not lease or rec.released: + return + rec.released = True + if not rec.readers: + self._finish_release(rec) + + def copy_prefix_into(self, lease: PrefixLease, end: int, + dest: Mapping[str, torch.Tensor]) -> None: + with self._lock: + rec = self._checked_lease(lease, end) + if set(dest) != {f.name for f in self.schema.features}: + raise CaptureContractError("destination does not contain the complete schema") + rows = {int(t.shape[0]) if isinstance(t, torch.Tensor) and t.ndim == 2 else -1 for t in dest.values()} + if len(rows) != 1 or next(iter(rows)) < end: + raise CaptureContractError("destination cannot hold the leased prefix") + self.schema.validate(dest, next(iter(rows)), "cpu") + entries = rec.entries[: (end + self.chunk_tokens - 1) // self.chunk_tokens] + if any(not self._complete(e) for e in entries): + raise CaptureContractError("leased cache block is missing or corrupt") + rec.readers += 1 + failure = None + try: + spans = [] + for entry in entries: + start, stop = entry.key.start, min(end, entry.key.end) + if entry.owner is not None and spans and spans[-1][2] is entry.owner: + old_start, _, owner, _ = spans[-1] + spans[-1] = (old_start, stop, owner, None) + else: + spans.append((start, stop, entry.owner, entry.features)) + with torch.no_grad(): + for start, stop, owner, block in spans: + for spec in self.schema.features: + source = (owner.features[spec.name][start:stop] if owner is not None + else block[spec.name][:stop - start]) + dest[spec.name][start:stop].copy_(source) + # Drop source views before releasing the reader's lease protection. + source = owner = block = None + spans.clear() + except BaseException as exc: + failure = _normalized_error(exc) + finally: + source = owner = block = None + spans.clear() + with self._lock: + rec.readers -= 1 + if rec.released and not rec.readers: + self._finish_release(rec) + if failure is not None: + raise failure from None + + def reserve_fill(self, key: ContentBlockKey, + execution: ExecutionIdentity) -> FillReservation | None: + self._validate_key(key) + if not isinstance(execution, ExecutionIdentity): + raise CaptureContractError("fill requires an execution identity") + with self._lock: + if key in self._entries or self._chunk_bytes > self.capacity_bytes: + self._fill_skips += 1 + return None + if not self._make_room_locked(self._chunk_bytes): + self._fill_skips += 1 + return None + handle = FillReservation(uuid.uuid4().hex, key, execution, self._chunk_bytes) + entry = _Entry(key, self._chunk_bytes) + self._entries[key] = entry + self._fills[handle.reservation_id] = _Fill(handle, entry) + self._bytes += self._chunk_bytes + self._peak_bytes = max(self._peak_bytes, self._bytes) + self._record_logical_peak_locked() + return handle + + def _record_logical_peak_locked(self): + self._logical_peak_bytes = max(self._logical_peak_bytes, + len(self._entries) * self._chunk_bytes) + + def _remove_entry_locked(self, entry): + entry.features.clear() + self._entries.pop(entry.key, None) + self._lru.pop(entry.key, None) + owner = entry.owner + if owner is None: + self._bytes -= entry.nbytes + else: + owner.keys.discard(entry.key) + if not owner.keys: + # Drop the final owner references before making storage budget + # available. A chunk view never charges only its logical rows. + owner.features.clear() + self._bytes -= owner.nbytes + self._owned_bytes -= owner.nbytes + self._owned_groups -= 1 + entry.owner = None + + def _make_room_locked(self, amount): + if amount > self.capacity_bytes: + return False + while self._bytes + amount > self.capacity_bytes: + victim = None + for key in self._lru: + entry = self._entries[key] + if entry.leases: + continue + if entry.owner is not None and entry.owner.leases: + continue + victim = entry + break + if victim is None: + return False + # Evict an entire owner group. Partial eviction cannot free its + # backing allocation and only creates holes in prefix coverage. + keys = tuple(victim.owner.keys) if victim.owner is not None else (victim.key,) + for key in keys: + self._remove_entry_locked(self._entries[key]) + self._evictions += 1 + return True + + def _adopt_released_sample(self, keys, features) -> bool: + """Take exclusive CPU ownership after every MaterializedSample reader ends. + + Only the controller calls this internal path. Key hashes are prepared + before release. No CPU tensor allocation/copy occurs here; every chunk + view belongs to a group charged for its complete underlying storages, + including any reused prefix, padding, and incomplete tail. + """ + if not keys: + return False + rows = next(iter(features.values())).shape[0] + self.schema.validate(features, rows, "cpu") + storages = {(str(t.device), t.untyped_storage().data_ptr()): t.untyped_storage().nbytes() + for t in features.values()} + nbytes = sum(storages.values()) + owner = _StorageOwner(nbytes, dict(features)) + pending = [] + failure = None + published = charged = False + entry = tensor = None + try: + with self._lock: + if not any(key not in self._entries for key in keys): + self._adoption_skips += 1 + self._fill_skips += len(keys) + elif not self._make_room_locked(nbytes): + self._adoption_skips += 1 + self._fill_skips += len(keys) + else: + # Capacity eviction may itself have removed these keys. + # Register every view with pending before constructing the + # next one, so partial metadata errors drop all references. + for key in keys: + if key not in self._entries: + entry = _Entry(key, self._chunk_bytes, "READY", owner=owner) + pending.append(entry) + for name, tensor in features.items(): + entry.features[name] = tensor[key.start:key.end] + owner.keys.update(entry.key for entry in pending) + self._bytes += nbytes + self._peak_bytes = max(self._peak_bytes, self._bytes) + self._owned_bytes += nbytes + self._owned_groups += 1 + charged = True + self._owned_peak_bytes = max(self._owned_peak_bytes, self._owned_bytes) + for entry in pending: + self._entries[entry.key] = entry + self._lru[entry.key] = None + self._record_logical_peak_locked() + self._adopted_samples += 1 + self._fill_skips += len(keys) - len(pending) + published = True + except BaseException as exc: + failure = _normalized_error(exc) + finally: + tensor = entry = None + if not published: + with self._lock: + for entry in pending: + if self._entries.get(entry.key) is entry: + self._entries.pop(entry.key) + self._lru.pop(entry.key, None) + entry.features.clear() + entry.owner = None + if charged: + self._bytes -= nbytes + self._owned_bytes -= nbytes + self._owned_groups -= 1 + pending.clear() + owner.features.clear() + owner.keys.clear() + entry = None + if failure is not None: + raise failure from None + return published + + def _drop_fill(self, rec: _Fill) -> None: + if self._entries.get(rec.entry.key) is rec.entry: + self._remove_entry_locked(rec.entry) + self._fills.pop(rec.handle.reservation_id, None) + + def publish_ready(self, fill: FillReservation, + features: Mapping[str, torch.Tensor]) -> bool: + with self._lock: + rec = self._fills.get(getattr(fill, "reservation_id", None)) + if rec is None or rec.handle is not fill or rec.writer_active or rec.abort_requested: + return False + rec.writer_active = True + owned = {} + tensor = None + error = None + try: + self.schema.validate(features, self.chunk_tokens, "cpu") + with torch.no_grad(): + for spec in self.schema.features: + tensor = torch.empty((self.chunk_tokens, spec.width), dtype=features[spec.name].dtype, device="cpu") + tensor.copy_(features[spec.name].detach()) + owned[spec.name] = tensor + # No caller tensor/view becomes durable cache storage. + except BaseException as exc: + error = exc + # A caught exception traceback can retain a failed copy's arguments. + # Normalize it and drop all owned locals before reusing WRITING bytes. + failure = _normalized_error(error) if error is not None else None + error = None + tensor = None + with self._lock: + rec.writer_active = False + if failure is not None or rec.abort_requested: + owned.clear() + self._drop_fill(rec) + if failure is not None: + raise failure from None + return False + rec.entry.features = owned + rec.entry.state = "READY" + self._fills.pop(fill.reservation_id) + self._lru[fill.key] = None + return True + + def abort_fill(self, fill: FillReservation) -> None: + with self._lock: + rec = self._fills.get(getattr(fill, "reservation_id", None)) + if rec is None or rec.handle is not fill: + return + if rec.writer_active: + rec.abort_requested = True + else: + self._drop_fill(rec) + + def stats(self) -> dict[str, int]: + with self._lock: + ready_bytes = sum(e.nbytes for e in self._entries.values() + if e.state == "READY" and e.owner is None) + ready_bytes += self._owned_bytes + return { + "ready_blocks": sum(e.state == "READY" for e in self._entries.values()), + "writing_blocks": sum(e.state == "WRITING" for e in self._entries.values()), + "ready_bytes": ready_bytes, + "writing_bytes": sum(e.nbytes for e in self._entries.values() if e.state == "WRITING"), + "cache_bytes": self._bytes, "cache_peak_bytes": self._peak_bytes, + "cache_logical_bytes": len(self._entries) * self._chunk_bytes, + "cache_logical_peak_bytes": self._logical_peak_bytes, + "cache_owned_bytes": self._owned_bytes, + "cache_owned_groups": self._owned_groups, + "cache_owned_peak_bytes": self._owned_peak_bytes, + "cache_adopted_samples": self._adopted_samples, + "cache_adoption_skips": self._adoption_skips, + "active_leases": len(self._leases), "fill_reservations": len(self._fills), + "leased_blocks": sum(e.leases > 0 for e in self._entries.values()), + "evictions": self._evictions, "fill_skips": self._fill_skips, + } diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py new file mode 100644 index 0000000000..4710f52988 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/config.py @@ -0,0 +1,41 @@ +"""Explicit native-plugin HS configuration, independent of legacy SG flags.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _flag(name: str) -> bool: + value = os.environ.get(name, "0") + if value not in ("0", "1"): + raise ValueError(f"{name} must be 0 or 1") + return value == "1" + + +@dataclass(frozen=True) +class HSConfig: + capture: bool = False + cache: bool = False + chunk_tokens: int = 256 + cache_bytes: int = 1 << 30 + pinned_bytes: int = 256 << 20 + output_bytes: int = 512 << 20 + device_bytes: int = 256 << 20 + max_workers: int = 4 + attempt_capacity: int = 4096 + + @classmethod + def from_env(cls) -> HSConfig: + values = {"capture": _flag("SPECFORGE_HS_CAPTURE"), + "cache": _flag("SPECFORGE_HS_CACHE")} + for name in ("chunk_tokens", "cache_bytes", "pinned_bytes", "output_bytes", + "device_bytes", "max_workers", "attempt_capacity"): + env = "SPECFORGE_HS_" + name.upper() + value = int(os.environ.get(env, str(getattr(cls, name)))) + if value <= 0: + raise ValueError(f"{env} must be positive") + values[name] = value + if values["cache"] and not values["capture"]: + raise ValueError("SPECFORGE_HS_CACHE=1 requires SPECFORGE_HS_CAPTURE=1") + return cls(**values) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py new file mode 100644 index 0000000000..8a8fb8b16f --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/controller.py @@ -0,0 +1,673 @@ +"""Bounded capture ownership, asynchronous D2H, full assembly and safe drain. + +B hands over compact independently owned suffix snapshots after admission. +Workers never touch a Req, execute a collective, or release sink-owned output. +""" + +from __future__ import annotations + +import os +import resource +import threading +import uuid +from collections import deque +from concurrent.futures import Future, ThreadPoolExecutor +from time import monotonic_ns, perf_counter +from dataclasses import dataclass, field + +import torch + +from .cache import PrefixFeatureCache, _token_tuple +from .protocol import ( + DTYPES, DTYPE_BYTES, AdmissionBudgetError, CaptureCancelledError, CaptureCompletion, + CaptureContractError, CapturePlan, CapturedSpan, DrainHandle, ExecutionIdentity, + ContentBlockKey, FeatureSchema, MaterializedSample, OutputReservation, PrefixLease, + canonical_digest, checked_int, _normalized_error, +) + + +@dataclass +class _Reservation: + handle: OutputReservation + submitted: bool = False + transient_released: bool = False + release_requested: bool = False + pinned_offset: int = 0 + + +@dataclass +class _Execution: + identity: ExecutionIdentity + reservations: set[str] = field(default_factory=set) + plan: CapturePlan | None = None + cancelled: bool = False + reason: str = "" + jobs: int = 0 + sample: MaterializedSample | None = None + owned_output: dict[str, torch.Tensor] = field(default_factory=dict) + cache_keys: tuple[ContentBlockKey, ...] = () + fills: dict[str, object] = field(default_factory=dict) + quarantined: list[object] = field(default_factory=list) + completion_pending: bool = False + drained_queued: bool = False + drained_polled: bool = False + drained: threading.Event = field(default_factory=threading.Event) + + +@dataclass +class _Job: + plan: CapturePlan + features: dict[str, torch.Tensor] + ready_event: object | None + is_cuda: bool + state: _Execution + future: Future + diagnostics: dict[str, int | float] = field(default_factory=dict) + + +class _UnsafeDrain(RuntimeError): + """The runtime must retain storage when CUDA/producer drain is unproven.""" + + def __init__(self, error, retained): + super().__init__(f"capture storage quarantined: drain could not be proven: {error}") + self.retained = retained + + +class CaptureController: + def __init__(self, cache: PrefixFeatureCache, schema: FeatureSchema, + pinned_budget_bytes: int = 256 * 1024**2, + output_budget_bytes: int = 512 * 1024**2, + device_budget_bytes: int = 256 * 1024**2, *, + max_workers: int = 4): + if not isinstance(cache, PrefixFeatureCache) or schema != cache.schema: + raise CaptureContractError("controller/cache schema mismatch") + self.cache, self.schema = cache, schema + self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" + self._diagnostic_every = max(1, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_EVERY", "1"))) + self._diagnostic_limit = max(0, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_MAX", "4096"))) + self._diagnostic_sequence = 0 + self._limits = { + "pinned": checked_int(pinned_budget_bytes, "pinned_budget_bytes"), + "output": checked_int(output_budget_bytes, "output_budget_bytes"), + "device": checked_int(device_budget_bytes, "device_budget_bytes"), + } + # PyTorch rounds pinned allocations up to a power of two and retains + # freed size buckets. One owned arena prevents cumulative idle buckets + # from bypassing the configured hard bound. Default budgets are exact + # powers of two; other budgets conservatively expose a smaller arena. + self._pinned_configured_bytes = self._limits["pinned"] + self._limits["pinned"] = (1 << (pinned_budget_bytes.bit_length() - 1) + if pinned_budget_bytes else 0) + self._pinned_arena = None + self._slot_alignment = max(DTYPE_BYTES[f.dtype] for f in schema.features) + self._pinned_free = [(0, self._limits["pinned"])] + self._max_workers = checked_int(max_workers, "max_workers", 1) + self._arena_poisoned = False + self._current = dict.fromkeys(self._limits, 0) + self._peaks = dict.fromkeys(self._limits, 0) + self._lock = threading.RLock() + self._reservations: dict[str, _Reservation] = {} + self._executions: dict[ExecutionIdentity, _Execution] = {} + self._completed: deque[CaptureCompletion] = deque() + self._drained: deque[ExecutionIdentity] = deque() + self._max_pending = max(1, min(4096, output_budget_bytes // schema.row_bytes)) + self._closed = False + self._fill_failures = 0 + # Admission attempts include retries; these are not unique-request counts. + # Reasons may overlap when more than one budget is currently exhausted. + self._admission_stats = dict.fromkeys(( + "admission_attempts", "admission_reserved_attempts", + "admission_budget_declined_attempts", "admission_hard_oversize_attempts", + "admission_declined_pinned_attempts", "admission_declined_output_attempts", + "admission_declined_device_attempts", "admission_declined_controller_max_pending_attempts", + "admission_declined_fragmentation_attempts", + ), 0) + self._adoption_calls = 0 + self._adoption_unused_skips = 0 + self._adoption_total_ms = self._adoption_max_ms = 0.0 + self._executor = ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="spec-feature-capture") + + def _staging_layout(self, rows): + offset, layout = 0, {} + for spec in self.schema.features: + itemsize = DTYPE_BYTES[spec.dtype] + offset = ((offset + itemsize - 1) // itemsize) * itemsize + size = rows * spec.width * itemsize + layout[spec.name] = (offset, size, spec) + offset += size + offset = ((offset + self._slot_alignment - 1) // self._slot_alignment) * self._slot_alignment + return offset, layout + + def _staging_views(self, rows, reservation=None): + required, layout = self._staging_layout(rows) + if required > self._limits["pinned"]: + raise AdmissionBudgetError("aligned suffix exceeds pinned arena capacity") + with self._lock: + if self._arena_poisoned: + raise CaptureContractError("pinned arena quarantined after unproven DMA drain") + offset = 0 + if reservation is not None: + rec = self._reservation(reservation) + if rec.transient_released or required > reservation.pinned_bytes: + raise CaptureContractError("staging slot is released or too small") + offset = rec.pinned_offset + elif self._current["pinned"]: + raise CaptureContractError("unreserved staging view while slots are active") + # Allocate once under the lock: workers must never race to create + # two retained power-of-two pinned allocations. + if self._pinned_arena is None: + self._pinned_arena = torch.empty((self._limits["pinned"],), dtype=torch.uint8, + device="cpu", pin_memory=True) + return {name: self._pinned_arena[offset + start:offset + start + size] + .view(DTYPES[spec.dtype]).view(rows, spec.width) + for name, (start, size, spec) in layout.items()} + + def _allocate_slot_locked(self, amount): + if not amount: + return 0 + for i, (start, size) in enumerate(self._pinned_free): + if size >= amount: + if size == amount: + self._pinned_free.pop(i) + else: + self._pinned_free[i] = (start + amount, size - amount) + return start + return None + + def _free_slot_locked(self, start, amount): + if not amount: + return + self._pinned_free.append((start, amount)) + merged = [] + for offset, size in sorted(self._pinned_free): + if merged and merged[-1][0] + merged[-1][1] == offset: + old, length = merged[-1] + merged[-1] = (old, length + size) + else: + merged.append((offset, size)) + self._pinned_free = merged + + def try_reserve_output(self, execution: ExecutionIdentity, N: int, L: int, + passthrough_bytes: int) -> OutputReservation | None: + if not isinstance(execution, ExecutionIdentity): + raise CaptureContractError("reservation requires execution identity") + checked_int(N, "N", 1) + checked_int(L, "L") + checked_int(passthrough_bytes, "passthrough_bytes") + if L > N: + raise CaptureContractError("L exceeds N") + charges = {"output": N * self.schema.row_bytes + passthrough_bytes, + "pinned": self._staging_layout(N - L)[0], + "device": (N - L) * self.schema.row_bytes} + with self._lock: + self._admission_stats["admission_attempts"] += 1 + for kind, amount in charges.items(): + if amount > self._limits[kind]: + self._admission_stats["admission_hard_oversize_attempts"] += 1 + raise AdmissionBudgetError(f"request needs {amount} {kind} bytes; hard limit {self._limits[kind]}") + if self._closed: + raise CaptureContractError("capture controller is shut down") + if self._arena_poisoned: + raise CaptureContractError("capture staging quarantined; restart required") + if execution in self._executions: + raise CaptureContractError("execution identity already active; retry needs a new execution epoch") + reasons = [kind for kind in charges + if self._current[kind] + charges[kind] > self._limits[kind]] + if len(self._executions) >= self._max_pending: + reasons.append("controller_max_pending") + if reasons: + self._admission_stats["admission_budget_declined_attempts"] += 1 + for reason in reasons: + self._admission_stats[f"admission_declined_{reason}_attempts"] += 1 + return None + slot = self._allocate_slot_locked(charges["pinned"]) + if slot is None: + self._admission_stats["admission_budget_declined_attempts"] += 1 + self._admission_stats["admission_declined_fragmentation_attempts"] += 1 + return None # Fragmentation cannot authorize overlapping DMA slots. + handle = OutputReservation(uuid.uuid4().hex, execution, charges["output"], + charges["pinned"], charges["device"], N, L, passthrough_bytes) + self._reservations[handle.reservation_id] = _Reservation(handle, pinned_offset=slot) + state = _Execution(execution, {handle.reservation_id}) + self._executions[execution] = state + self._admission_stats["admission_reserved_attempts"] += 1 + for kind, amount in charges.items(): + self._current[kind] += amount + self._peaks[kind] = max(self._peaks[kind], self._current[kind]) + return handle + + def _reservation(self, handle: OutputReservation) -> _Reservation: + rec = self._reservations.get(getattr(handle, "reservation_id", None)) + if rec is None or rec.handle is not handle: + raise CaptureContractError("output reservation is foreign, stale or released") + return rec + + def freeze_plan(self, execution: ExecutionIdentity, namespace: str, tokens: tuple[int, ...], + N: int, L: int, lease: PrefixLease, + reservation: OutputReservation) -> CapturePlan: + _token_tuple(tokens) + if len(tokens) != N: + raise CaptureContractError("plan tokens do not match N") + self.cache._validate_plan_lease(lease, L, namespace, tokens) + digest = canonical_digest(["spec-plan-v1", execution.server_epoch, execution.request_id, + execution.attempt_id, execution.execution_epoch, namespace, self.schema.signature, + N, L, tokens]) + with self._lock: + rec = self._reservation(reservation) + state = self._executions.get(execution) + if (reservation.execution != execution or reservation.N != N or reservation.L != L + or state is None or state.cancelled or rec.submitted): + raise CaptureContractError("plan/reservation execution or range mismatch") + if state.plan is not None: + raise CaptureContractError("execution already has a frozen plan") + plan = CapturePlan(execution, namespace, tokens, N, L, self.schema.signature, + lease, reservation, digest) + state.plan = plan + return plan + + def _release_transient_locked(self, rec: _Reservation) -> None: + if rec.transient_released: + return + rec.transient_released = True + self._free_slot_locked(rec.pinned_offset, rec.handle.pinned_bytes) + self._current["pinned"] -= rec.handle.pinned_bytes + self._current["device"] -= rec.handle.device_bytes + + def _release_host_locked(self, rec: _Reservation, state: _Execution) -> None: + self._current["output"] -= rec.handle.host_bytes + self._reservations.pop(rec.handle.reservation_id, None) + state.reservations.discard(rec.handle.reservation_id) + assert all(v >= 0 for v in self._current.values()) + + def _maybe_drained_locked(self, state: _Execution) -> None: + if state.jobs or state.reservations or state.fills or state.quarantined: + return + state.drained.set() + if not state.drained_queued: + self._drained.append(state.identity) + state.drained_queued = True + self._reap_locked(state) + + def _reap_locked(self, state: _Execution) -> None: + if state.drained_polled and state.drained.is_set() and not state.completion_pending: + if self._executions.get(state.identity) is state: + self._executions.pop(state.identity) + + def release_output(self, reservation: OutputReservation) -> None: + with self._lock: + rec = self._reservations.get(getattr(reservation, "reservation_id", None)) + if rec is None or rec.handle is not reservation: + return + rec.release_requested = True + if rec.submitted: + # The worker/sink remains the owner. No timeout can revoke its read. + return + state = self._executions[reservation.execution] + if state.plan is not None: + self.cache.release_lease(state.plan.lease) + self._release_transient_locked(rec) + self._release_host_locked(rec, state) + self._maybe_drained_locked(state) + + def _sample_released(self, reservation: OutputReservation) -> None: + with self._lock: + rec = self._reservations.get(reservation.reservation_id) + if rec is None or rec.handle is not reservation: + return + state = self._executions[reservation.execution] + state.sample = None + owned, state.owned_output = state.owned_output, {} + adoption_started = perf_counter() + try: + # MaterializedSample.release has already removed the sink's + # tensor references. Readers may not access or modify them + # after this ownership transfer. Keep the output charge until + # the cache has charged every retained backing allocation. + if owned and not state.cancelled and not rec.release_requested and not self._closed: + if state.cache_keys or not self.cache.skip_unused_cache_work: + self._adoption_calls += 1 + self.cache._adopt_released_sample(state.cache_keys, owned) + else: + self._adoption_unused_skips += 1 + except Exception: + self._fill_failures += 1 + finally: + elapsed_ms = (perf_counter() - adoption_started) * 1000.0 + self._adoption_total_ms += elapsed_ms + self._adoption_max_ms = max(self._adoption_max_ms, elapsed_ms) + state.cache_keys = () + owned.clear() + self._release_transient_locked(rec) + self._release_host_locked(rec, state) + self._maybe_drained_locked(state) + + def submit(self, plan: CapturePlan, span: CapturedSpan) -> Future[MaterializedSample]: + if not isinstance(plan, CapturePlan) or not isinstance(span, CapturedSpan): + raise CaptureContractError("submit requires a frozen plan and span") + if span.execution != plan.execution or (span.start, span.end) != (plan.L, plan.N): + raise CaptureContractError("captured span must be the plan's absolute [L,N)") + if plan.schema_signature != self.schema.signature: + raise CaptureContractError("plan schema mismatch") + features = dict(span.features) + self.schema.validate(features, plan.N - plan.L) + devices = {tensor.device for tensor in features.values()} + if len(devices) != 1 or next(iter(devices)).type not in ("cpu", "cuda"): + raise CaptureContractError("all suffix features must share one CPU/CUDA device") + is_cuda = next(iter(devices)).type == "cuda" + if is_cuda and span.ready_event is None: + raise CaptureContractError("CUDA snapshots require a producer ready event") + if span.ready_event is not None and not callable(getattr(span.ready_event, "synchronize", None)): + raise CaptureContractError("ready_event must support synchronize") + # Count underlying storage, not tensor-view rows. B must hand over compact + # owned snapshots; a tiny suffix view of a full padded batch is rejected. + storages = {(str(t.device), t.untyped_storage().data_ptr()): t.untyped_storage().nbytes() + for t in features.values()} + source_bytes = sum(storages.values()) + if source_bytes > plan.reservation.device_bytes: + raise CaptureContractError("snapshot underlying storage exceeds the admission reservation") + with self._lock: + rec = self._reservation(plan.reservation) + state = self._executions.get(plan.execution) + if (state is None or state.plan is not plan or state.cancelled or rec.release_requested + or rec.submitted or self._closed): + raise CaptureContractError("execution is stale, cancelled, submitted or closed") + self.cache._validate_plan_lease(plan.lease, plan.L, plan.namespace, plan.tokens) + rec.submitted = True + state.jobs += 1 + future = Future() + # Future.cancel() cannot cancel the cleanup worker and strand storage. + future.set_running_or_notify_cancel() + job = _Job(plan, features, span.ready_event, is_cuda, state, future) + if self._diagnostics: + sequence = self._diagnostic_sequence + self._diagnostic_sequence += 1 + if sequence % self._diagnostic_every == 0 and sequence // self._diagnostic_every < self._diagnostic_limit: + job.diagnostics["p0_v1_materialize_submitted_ns"] = monotonic_ns() + try: + self._executor.submit(self._run_job, job) + except BaseException: + rec.submitted = False + state.jobs -= 1 + raise + return future + + def _cancelled(self, job: _Job) -> bool: + with self._lock: + rec = self._reservations.get(job.plan.reservation.reservation_id) + return job.state.cancelled or rec is None or rec.release_requested + + def _copy_suffix(self, job: _Job) -> dict[str, torch.Tensor]: + """Only this worker owns these sources/staging until the real copy drain.""" + if not job.is_cuda: + if job.ready_event is not None: + try: + ready_started = monotonic_ns() if job.diagnostics else 0 + job.ready_event.synchronize() + if job.diagnostics: + job.diagnostics["p0_v1_producer_ready_wait_ms"] = (monotonic_ns() - ready_started) / 1e6 + except BaseException as exc: + raise _UnsafeDrain(exc, [job.ready_event]) from exc + return {} if self._cancelled(job) else dict(job.features) + staging = {} + stream = None + source = target = None + device = next(iter(job.features.values())).device + try: + with torch.cuda.device(device): + if self._cancelled(job): + job.ready_event.synchronize() + return {} + with self._lock: + poisoned = self._arena_poisoned + if poisoned: + # A previous failed DMA still owns this shared landing + # buffer. Drain this job's producer, but never touch arena. + job.ready_event.synchronize() + raise CaptureContractError("pinned arena quarantined after unproven DMA drain") + stream = torch.cuda.Stream(device=device) + with torch.cuda.stream(stream): + if job.diagnostics: + waiting = torch.cuda.Event(enable_timing=True) + ready = torch.cuda.Event(enable_timing=True) + copy_start = torch.cuda.Event(enable_timing=True) + waiting.record(stream) + stream.wait_event(job.ready_event) + if job.diagnostics: + ready.record(stream) + alloc_started = monotonic_ns() + staging = self._staging_views(job.plan.N - job.plan.L, job.plan.reservation) + if job.diagnostics: + job.diagnostics["p0_v1_staging_alloc_ms"] = (monotonic_ns() - alloc_started) / 1e6 + copy_start.record(stream) + for name, source in job.features.items(): + target = staging[name] + target.copy_(source, non_blocking=True) + source.record_stream(stream) + copied = torch.cuda.Event(enable_timing=True) if job.diagnostics else torch.cuda.Event() + copied.record(stream) + copied.synchronize() + if job.diagnostics: + # Read events only after the existing real copy drain. No + # timing-only synchronization is introduced on the hot path. + job.diagnostics["p0_v1_producer_ready_wait_ms"] = waiting.elapsed_time(ready) + job.diagnostics["p0_v1_d2h_copy_ms"] = copy_start.elapsed_time(copied) + return staging + except BaseException as exc: + drain_errors = [] + # A failed wait_event may leave the copy stream independent of the + # producer. Drain BOTH owners, even if either synchronize raises. + if stream is not None: + try: + stream.synchronize() + except BaseException as drain_error: + drain_errors.append(drain_error) + try: + job.ready_event.synchronize() + except BaseException as drain_error: + drain_errors.append(drain_error) + if drain_errors: + raise _UnsafeDrain(drain_errors[0], [staging, stream, job.ready_event]) from exc + source = target = None + staging.clear() + raise + + def _prepare_cache_adoption(self, job: _Job) -> tuple: + # Hash/construct keys on the assembly worker, never in the sink release + # callback (which may run on the scheduler thread). + if self.cache.skip_unused_cache_work and ( + self.cache.capacity_bytes == 0 or job.plan.N < self.cache.chunk_tokens): + return () + try: + return self.cache.block_keys(job.plan.namespace, job.plan.tokens) + except Exception: + with self._lock: + self._fill_failures += 1 + return () + + def _run_job(self, job: _Job) -> None: + output, staging = {}, {} + sample = None + completion = None + unsafe = False + try: + if job.diagnostics: + started_ns = monotonic_ns() + job.diagnostics["p0_v1_materialize_started_ns"] = started_ns + job.diagnostics["p0_v1_d2h_queue_ms"] = (started_ns - job.diagnostics["p0_v1_materialize_submitted_ns"]) / 1e6 + job.diagnostics["p0_v1_d2h_bytes"] = (job.plan.N - job.plan.L) * self.schema.row_bytes if job.is_cuda else 0 + copy_started = perf_counter() + staging = self._copy_suffix(job) + copied_at = perf_counter() + job.features.clear() + job.ready_event = None + if self._cancelled(job): + raise CaptureCancelledError(job.state.reason or "execution released") + with torch.no_grad(): + if job.diagnostics: + alloc_started = monotonic_ns() + faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt + for spec in self.schema.features: + output[spec.name] = torch.empty((job.plan.N, spec.width), dtype=DTYPES[spec.dtype], device="cpu") + if job.diagnostics: + prefix_started = monotonic_ns() + job.diagnostics["p0_v1_output_alloc_ms"] = (prefix_started - alloc_started) / 1e6 + job.diagnostics["p0_v1_output_alloc_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started + faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt + try: + self.cache.copy_prefix_into(job.plan.lease, job.plan.L, output) + finally: + self.cache.release_lease(job.plan.lease) + if job.diagnostics: + suffix_started = monotonic_ns() + job.diagnostics["p0_v1_prefix_copy_ms"] = (suffix_started - prefix_started) / 1e6 + job.diagnostics["p0_v1_prefix_copy_bytes"] = job.plan.L * self.schema.row_bytes + job.diagnostics["p0_v1_prefix_copy_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started + faults_started = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt + for name in output: + output[name][job.plan.L:job.plan.N].copy_(staging[name]) + if job.diagnostics: + job.diagnostics["p0_v1_suffix_copy_ms"] = (monotonic_ns() - suffix_started) / 1e6 + job.diagnostics["p0_v1_suffix_copy_bytes"] = (job.plan.N - job.plan.L) * self.schema.row_bytes + job.diagnostics["p0_v1_suffix_copy_minor_faults"] = resource.getrusage(resource.RUSAGE_THREAD).ru_minflt - faults_started + staging.clear() + with self._lock: + self._release_transient_locked(self._reservation(job.plan.reservation)) + if self._cancelled(job): + raise CaptureCancelledError(job.state.reason or "execution released") + self.schema.validate(output, job.plan.N, "cpu") + assembled_at = perf_counter() + cache_keys = self._prepare_cache_adoption(job) + filled_at = perf_counter() + if job.diagnostics: + job.diagnostics["p0_v1_key_prepare_ms"] = (filled_at - assembled_at) * 1000.0 + # Assembly and key preparation are complete. Sample/completion + # bookkeeping and future delivery follow on this same worker. + job.diagnostics["p0_v1_materialize_done_ns"] = monotonic_ns() + sample = MaterializedSample(job.plan.execution, output, job.plan.N, + lambda: self._sample_released(job.plan.reservation), timings={ + **job.diagnostics, + "d2h_wait_ms": (copied_at - copy_started) * 1000.0, + "assembly_ms": (assembled_at - copied_at) * 1000.0, + "cache_fill_ms": (filled_at - assembled_at) * 1000.0, + }) + with self._lock: + job.state.owned_output = dict(output) + job.state.cache_keys = cache_keys + output.clear() + if self._cancelled(job): + sample.release() + sample = None + raise CaptureCancelledError(job.state.reason or "execution released") + job.state.sample = sample + completion = CaptureCompletion(job.plan.execution, sample=sample) + except _UnsafeDrain as exc: + unsafe = True + with self._lock: + # A persistent arena must never be reused while any old DMA + # may still access it. Also stop fresh admission immediately. + self._arena_poisoned = True + # Retain sources AND landing buffers. Keep every charge until a + # real recovery mechanism proves drain; timeout is never success. + job.state.quarantined.extend([job, exc.retained]) + completion = CaptureCompletion(job.plan.execution, error=RuntimeError(str(exc))) + except BaseException as exc: + completion = CaptureCompletion(job.plan.execution, error=_normalized_error(exc)) + finally: + self.cache.release_lease(job.plan.lease) + output.clear() + if not unsafe: + staging.clear() + job.features.clear() + job.ready_event = None + with self._lock: + rec = self._reservations.get(job.plan.reservation.reservation_id) + if rec is not None: + self._release_transient_locked(rec) + if sample is None: + self._release_host_locked(rec, job.state) + with self._lock: + job.state.jobs -= 1 + job.state.completion_pending = True + self._completed.append(completion) + self._maybe_drained_locked(job.state) + # Publish the result only after completion bookkeeping and resources. + if completion.error is not None: + job.future.set_exception(completion.error) + else: + job.future.set_result(completion.sample) + + def poll_completed(self) -> list[CaptureCompletion]: + with self._lock: + result = list(self._completed) + self._completed.clear() + for completion in result: + state = self._executions.get(completion.execution) + if state is not None: + state.completion_pending = False + self._reap_locked(state) + return result + + def invalidate(self, execution: ExecutionIdentity, reason: str) -> DrainHandle: + with self._lock: + state = self._executions.get(execution) + if state is None: + event = threading.Event() + event.set() + return DrainHandle(event) + state.cancelled, state.reason = True, str(reason) + for fill in tuple(state.fills.values()): + self.cache.abort_fill(fill) + for token in tuple(state.reservations): + rec = self._reservations.get(token) + if rec is not None and not rec.submitted: + self.release_output(rec.handle) + self._maybe_drained_locked(state) + return DrainHandle(state.drained) + + def poll_drained(self) -> list[ExecutionIdentity]: + with self._lock: + result = list(self._drained) + self._drained.clear() + for execution in result: + state = self._executions.get(execution) + if state is not None: + state.drained_polled = True + self._reap_locked(state) + return result + + def stats(self) -> dict[str, int | float]: + result = self.cache.stats() + with self._lock: + result.update(self._admission_stats) + result.update({f"{k}_bytes": v for k, v in self._current.items()}) + result.update({f"{k}_peak_bytes": v for k, v in self._peaks.items()}) + result.update(pending_executions=len(self._executions), + pending_copies=sum(s.jobs for s in self._executions.values()), + output_reservations=len(self._reservations), + quarantined_executions=sum(bool(s.quarantined) for s in self._executions.values()), + fill_failures=self._fill_failures, + capture_workers=self._max_workers, + cache_adoption_calls=self._adoption_calls, + cache_adoption_unused_skips=self._adoption_unused_skips, + cache_adoption_total_ms=self._adoption_total_ms, + cache_adoption_max_ms=self._adoption_max_ms, + pinned_free_bytes=sum(size for _, size in self._pinned_free), + pinned_free_slots=len(self._pinned_free)) + result.update(pinned_configured_bytes=self._pinned_configured_bytes, + pinned_capacity_bytes=self._limits["pinned"], + pinned_arena_bytes=(self._pinned_arena.numel() if self._pinned_arena is not None else 0), + pinned_arena_poisoned=int(self._arena_poisoned)) + return result + + def shutdown(self, wait: bool = True) -> None: + with self._lock: + self._closed = True + executions = tuple(self._executions) + for execution in executions: + self.invalidate(execution, "controller shutdown") + # Never cancel queued cleanup workers. They retain producer events/storage. + self._executor.shutdown(wait=wait, cancel_futures=False) + if wait: + with self._lock: + if any(not state.drained.is_set() for state in self._executions.values()): + raise RuntimeError("capture shutdown not drained: sink output or quarantined storage still owned") diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py new file mode 100644 index 0000000000..3eed995c8f --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/http.py @@ -0,0 +1,30 @@ +"""HTTP forwarding only; all HS state changes execute on TP schedulers.""" + +from __future__ import annotations + +from fastapi import Request +from fastapi.responses import JSONResponse + + +def install_control_route(result, *args, **kwargs): + from sglang.srt.entrypoints import http_server + + app = http_server.app + if any(getattr(route, "path", None) == "/spec_capture/control" for route in app.routes): + return result + + async def spec_capture_control(request: Request): + body = await request.json() + if not isinstance(body, dict): + return JSONResponse({"error": "capture control requires a JSON object"}, status_code=400) + manager = http_server.get_global_state().tokenizer_manager + try: + # plugin_control owns the request/reply communicator. It invokes + # runtime.control on every scheduler rank in the same phase order. + reply = await manager.plugin_control("specforge_hs", body) + except (ValueError, RuntimeError) as exc: + return JSONResponse({"error": str(exc)}, status_code=409) + return JSONResponse(reply) + + app.add_api_route("/spec_capture/control", spec_capture_control, methods=["POST"]) + return result diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py new file mode 100644 index 0000000000..2c6feaa553 --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/protocol.py @@ -0,0 +1,251 @@ +"""Plan A v1 values, ownership handles, and canonical feature validation. + +Only aux and post-norm last_hidden artifacts are runtime feature names. Sink +object names belong to the bridge. ContentBlockKey._tokens is frozen per-block +metadata (root-approved v1 clarification), never a wire field or lookup side table. +""" + +from __future__ import annotations + +import hashlib +import json +import threading +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Callable, Mapping + +import torch + +DTYPES = { + "bfloat16": torch.bfloat16, + "float16": torch.float16, + "float32": torch.float32, + "float64": torch.float64, +} +DTYPE_BYTES = {"bfloat16": 2, "float16": 2, "float32": 4, "float64": 8} + + +class CaptureContractError(ValueError): + """A malformed plan/span/schema, rather than a recoverable cache miss.""" + + +class AdmissionBudgetError(CaptureContractError): + """One request cannot fit even when all other reservations are released.""" + + +class CaptureCancelledError(RuntimeError): + """Execution fenced; all safely reclaimable resources have been drained.""" + + +def checked_int(value: int, name: str, minimum: int = 0) -> int: + if type(value) is not int or value < minimum: + raise CaptureContractError(f"{name} must be an integer >= {minimum}") + return value + + +def canonical_digest(value) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("ascii")).hexdigest() + + +def _normalized_error(error: BaseException) -> BaseException: + """A safe error outcome cannot retain drained tensor/stream traceback locals.""" + try: + clean = type(error)(str(error)) + except BaseException: + clean = RuntimeError(f"{type(error).__name__}: {error}") + clean.__traceback__ = None + clean.__cause__ = None + clean.__context__ = None + clean.__suppress_context__ = True + return clean + + +@dataclass(frozen=True) +class FeatureSpec: + name: str + width: int + dtype: str + layer_ids: tuple[int, ...] + + def __post_init__(self): + if self.name not in ("aux", "last_hidden"): + raise CaptureContractError(f"unknown feature artifact {self.name!r}") + checked_int(self.width, "feature width", 1) + if self.dtype not in DTYPES: + raise CaptureContractError(f"unsupported canonical dtype {self.dtype!r}") + if not isinstance(self.layer_ids, tuple): + raise CaptureContractError("layer_ids must be an immutable tuple") + for layer in self.layer_ids: + checked_int(layer, "layer id") + if len(set(self.layer_ids)) != len(self.layer_ids): + raise CaptureContractError("duplicate capture layers") + if self.name == "aux" and not self.layer_ids: + raise CaptureContractError("aux needs ordered capture layer ids") + if self.name == "last_hidden" and self.layer_ids: + raise CaptureContractError("post-norm last_hidden uses layer_ids=()") + + +@dataclass(frozen=True) +class FeatureSchema: + features: tuple[FeatureSpec, ...] + signature: str = field(init=False) + row_bytes: int = field(init=False) + + def __post_init__(self): + if not isinstance(self.features, tuple) or not self.features: + raise CaptureContractError("features must be a nonempty immutable tuple") + if any(not isinstance(f, FeatureSpec) for f in self.features): + raise CaptureContractError("schema requires FeatureSpec entries") + if len({f.name for f in self.features}) != len(self.features): + raise CaptureContractError("duplicate feature artifact") + manifest = [(f.name, f.width, f.dtype, f.layer_ids) for f in self.features] + object.__setattr__(self, "signature", canonical_digest(["spec-features-v1", manifest])) + object.__setattr__(self, "row_bytes", sum(f.width * DTYPE_BYTES[f.dtype] for f in self.features)) + + def validate(self, features: Mapping[str, torch.Tensor], rows: int, + device: str | None = None) -> None: + checked_int(rows, "feature rows") + if set(features) != {f.name for f in self.features}: + raise CaptureContractError("feature names do not match the complete schema") + for spec in self.features: + tensor = features[spec.name] + if not isinstance(tensor, torch.Tensor): + raise CaptureContractError(f"{spec.name} is not a tensor") + if tuple(tensor.shape) != (rows, spec.width): + raise CaptureContractError(f"{spec.name} shape {tuple(tensor.shape)} != {(rows, spec.width)}") + if tensor.dtype != DTYPES[spec.dtype]: + raise CaptureContractError(f"{spec.name} dtype does not match {spec.dtype}") + if device is not None and tensor.device.type != device: + raise CaptureContractError(f"{spec.name} must be on {device}") + if not tensor.is_contiguous(): + raise CaptureContractError(f"{spec.name} must be contiguous") + + +@dataclass(frozen=True) +class ExecutionIdentity: + server_epoch: str + request_id: str + attempt_id: str + execution_epoch: int + + def __post_init__(self): + if not all(isinstance(v, str) and v for v in (self.server_epoch, self.request_id, self.attempt_id)): + raise CaptureContractError("execution identity fields must be nonempty strings") + checked_int(self.execution_epoch, "execution epoch") + + +@dataclass(frozen=True) +class ContentBlockKey: + namespace: str + start: int + end: int + digest: str + parent_digest: str + _tokens: tuple[int, ...] = field(repr=False) + + +@dataclass(frozen=True) +class PrefixLease: + lease_id: str + coverage_end: int + + +@dataclass(frozen=True) +class FillReservation: + reservation_id: str + key: ContentBlockKey + execution: ExecutionIdentity + nbytes: int + + +@dataclass(frozen=True) +class OutputReservation: + reservation_id: str + execution: ExecutionIdentity + host_bytes: int + pinned_bytes: int + device_bytes: int + N: int + L: int + passthrough_bytes: int + + +@dataclass(frozen=True) +class CapturePlan: + execution: ExecutionIdentity + namespace: str + tokens: tuple[int, ...] + N: int + L: int + schema_signature: str + lease: PrefixLease + reservation: OutputReservation + plan_digest: str + + +@dataclass(frozen=True) +class CapturedSpan: + execution: ExecutionIdentity + start: int + end: int + features: Mapping[str, torch.Tensor] + ready_event: torch.cuda.Event | None + + +class MaterializedSample: + """Owned CPU output; release only after every sink reader has ended. + + A release drops this object's tensor references before returning its budget. + Copies/views retained by the caller must have ended before invoking release. + """ + + def __init__(self, execution: ExecutionIdentity, features: Mapping[str, torch.Tensor], + n_tokens: int, on_release: Callable[[], None], *, timings=None): + self.execution = execution + self.n_tokens = n_tokens + self.timings = MappingProxyType(dict(timings or {})) + self._features = MappingProxyType(dict(features)) + self._on_release = on_release + self._lock = threading.Lock() + self._released = False + + @property + def features(self) -> Mapping[str, torch.Tensor]: + with self._lock: + if self._released: + raise CaptureContractError("sample already released") + return self._features + + def release(self) -> None: + with self._lock: + if self._released: + return + self._released = True + self._features = MappingProxyType({}) + callback, self._on_release = self._on_release, lambda: None + callback() + + +@dataclass(frozen=True) +class CaptureCompletion: + execution: ExecutionIdentity + sample: MaterializedSample | None = None + error: BaseException | None = None + + def __post_init__(self): + if (self.sample is None) == (self.error is None): + raise CaptureContractError("completion must have exactly one outcome") + + +class DrainHandle: + """A timeout means storage is still owned, never permission to reclaim.""" + + def __init__(self, event: threading.Event): + self._event = event + + def done(self) -> bool: + return self._event.is_set() + + def wait(self, timeout: float | None = None) -> bool: + return self._event.wait(timeout) diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py new file mode 100644 index 0000000000..b3e4168d0c --- /dev/null +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/hs/runtime.py @@ -0,0 +1,1093 @@ +"""Native capture admission and publication, owned by the scheduler thread. + +SG owns KV/SSM admission and TP transaction receipts. This module owns only HS +leases, output allocations and immutable external-attempt publication. Assembly +workers and sink callbacks never read a Req or execute a collective. +""" + +from __future__ import annotations + +import copy +import dataclasses +import hashlib +import json +import logging +import os +import threading +import time +import uuid +from collections import deque +from concurrent.futures import Future +from typing import Any + +import torch + +from .attempts import ( + Attempt, AttemptCapacityDeclined, AttemptRegistry, CaptureControlRejected, + digest_request, scalar_error, +) +from .cache import PrefixFeatureCache +from .config import HSConfig +from .controller import CaptureController +from .protocol import ( + CaptureContractError, CapturedSpan, ExecutionIdentity, FeatureSchema, + FeatureSpec, OutputReservation, PrefixLease, +) + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class _Candidate: + execution: ExecutionIdentity + entry: Attempt + tokens: tuple[int, ...] + lease: PrefixLease | None = None + reservation: OutputReservation | None = None + L: int = 0 + + +@dataclasses.dataclass +class _Publication: + execution: ExecutionIdentity + entry: Attempt + spec: dict + plan: Any + materialize: Future + output: Future + consumed: bool = False + sample: Any = None + sink: Future | None = None + sink_index: int = 0 + sink_count: int = 0 + cleanup: Future | None = None + failed: BaseException | None = None + publish_started: float = 0.0 + diagnostic: dict | None = None + + +class HSCompletion: + """Native ResponseCompletion for one immutable execution. + + Cancelling a response fences its execution. The runtime retains the job + until materialization and any shared sink reader really stop. + """ + + def __init__(self, runtime, job): + self.runtime, self.job = runtime, job + self._cancelled = self._resolved = False + + def done(self): + return self._cancelled or self.job.output.done() + + def result(self): + if self._cancelled or self._resolved: + return + self.runtime._assert_owner() + if not self.runtime._same_execution(self.job): + self._resolved = True + return + try: + result = self.job.output.result() + except Exception as exc: + result = {"sample_id": self.job.spec["sample_id"], "error": str(exc)} + if self.job.diagnostic is not None: + now = time.monotonic_ns() + self.job.diagnostic["p0_v1_response_callback_ns"] = now + observed = self.job.diagnostic.get("p0_v1_owner_observed_ns") + if observed is not None: + elapsed = (now - observed) / 1e6 + self.job.diagnostic["p0_v1_owner_to_response_callback_ms"] = elapsed + result.setdefault("timings", {})["p0_v1_owner_to_response_callback_ms"] = elapsed + self.runtime.capture.results[self.job.execution.request_id] = result + self._resolved = True + + def cancel(self): + if self._resolved: + return False + if not self._cancelled: + self.runtime._assert_owner() + self.runtime._cancel_job(self.job, "deferred-response-cancelled") + self._cancelled = True + return True + + +class HSCaptureRuntime: + control_namespace = "specforge_hs" + + def __init__(self, scheduler, capture, config: HSConfig, *, manifest=None, + schema=None, tp_size=None, tp_rank=0): + self.s, self.capture, self.config = scheduler, capture, config + self.capture.sink.require_hard_pin = True + self.writer, self.enabled = capture.is_writer, config.cache + self.owner_thread = threading.get_ident() + self.tp_size, self.tp_rank = tp_size, tp_rank + if self.tp_size is None: + from sglang.srt.runtime_context import get_parallel + parallel = get_parallel() + # Native capture elects the attention-TP writer from this same + # resolved topology; keep its TP rank as execution diagnostics. + self.tp_size, self.tp_rank = parallel.tp_size, parallel.tp_rank + self.epoch = self._writer_broadcast("startup.epoch", lambda: uuid.uuid4().hex) + if manifest is None or schema is None: + manifest, schema = self._local_phase("startup.model", self._model_identity) + + def identify(): + self.schema = schema + self.schema_wire = [dataclasses.asdict(feature) for feature in schema.features] + for feature in self.schema_wire: + feature["layer_ids"] = list(feature["layer_ids"]) + identity = {**manifest, "load_epoch": self.epoch, "schema": self.schema_wire, + "chunk_tokens": config.chunk_tokens} + self.signature = hashlib.sha256(json.dumps( + identity, sort_keys=True, separators=(",", ":"), default=str).encode()).hexdigest() + self.namespace = "spec-feature-v1:" + self.signature + self.registry = AttemptRegistry(self.epoch, config.attempt_capacity) + return self.signature + + signature = self._local_phase("startup.identity", identify) + self._agree("startup.signature", signature) + self.cache = self.controller = None + + def create_core(): + if self.writer: + self.cache = PrefixFeatureCache(schema, config.chunk_tokens, + config.cache_bytes if self.enabled else 0) + self.controller = CaptureController( + self.cache, schema, config.pinned_bytes, config.output_bytes, + config.device_bytes, max_workers=config.max_workers) + + try: + self._local_phase("startup.core", create_core) + except Exception: + if self.controller is not None: + self.controller.shutdown(wait=True) + raise + self.candidates: dict[str, _Candidate] = {} + self.jobs: dict[ExecutionIdentity, _Publication] = {} + self.history = deque(maxlen=256) + self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" + self._diagnostic_every = max(1, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_EVERY", "1"))) + self._diagnostic_limit = max(0, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_MAX", "4096"))) + self._diagnostic_sequence = 0 + self._diagnostic_batch = 0 + self.diagnostics = deque(maxlen=self._diagnostic_limit) + self._transferred_tokens = [] + self._closed = False + self.fatal_capture_error = None + self.quarantined_snapshots = [] + self.quarantined_publications = [] + + def _assert_owner(self): + if threading.get_ident() != self.owner_thread: + raise RuntimeError("HS state may only be changed on the scheduler thread") + + def _local_phase(self, phase, call): + """Vote local work BEFORE any subsequent internal collective. + + SG's outer callback phase runs too late to repair divergent nested + collectives. Labels also fail closed if ranks choose different plugin + branches that happen to arrive at a gather together. ``call`` must not + itself contain a collective; poll/lifecycle therefore remain local. + """ + value, error = None, None + try: + value = call() + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + votes = self.gather((phase, error)) + if any(not isinstance(vote, tuple) or len(vote) != 2 + or vote[0] != phase for vote in votes): + raise CaptureControlRejected(f"TP HS phase order differs at {phase}: {votes!r}") + if any(vote[1] is not None for vote in votes): + raise CaptureControlRejected(f"TP HS {phase} failed: {votes!r}") + return value + + def _agree(self, phase, value): + votes = self.gather((phase, value)) + if any(not isinstance(vote, tuple) or len(vote) != 2 + or vote[0] != phase or vote[1] != value for vote in votes): + raise CaptureControlRejected(f"TP HS {phase} differs: {votes!r}") + + def _writer_broadcast(self, phase, build): + payload = self._local_phase(phase + ".build", lambda: build() if self.writer else None) + return self.broadcast(payload) + + def _broadcast_wire(self, phase, entry): + wire = self._writer_broadcast(phase, lambda: self._wire(entry)) + + def apply(): + if any(wire.get(name) != expected for name, expected in ( + ("server_epoch", self.epoch), ("store_id", entry.identity[0]), + ("sample_id", entry.identity[1]), ("gen", entry.identity[2]), + ("attempt_id", entry.attempt_id), ("reservation_token", entry.token))): + raise CaptureContractError("TP HS wire identity changed") + entry.state = wire["state"] + + self._local_phase(phase + ".apply", apply) + return wire + + def broadcast(self, value): + if self.tp_size == 1: + return value + if hasattr(self.s, "prefix_admission_broadcast"): + return self.s.prefix_admission_broadcast(value) + import torch.distributed as dist + values = [value] + dist.broadcast_object_list(values, src=self.s.tp_group.ranks[0], + group=self.s.tp_cpu_group) + return values[0] + + def gather(self, value): + if self.tp_size == 1: + return [value] + if hasattr(self.s, "prefix_admission_gather"): + return self.s.prefix_admission_gather(value) + import torch.distributed as dist + values = [None] * self.tp_size + dist.all_gather_object(values, value, group=self.s.tp_cpu_group) + return values + + def _model_identity(self): + from sglang.srt.arg_groups.model_override_base import resolved_view + from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule + from specforge_sglang_capture.sink import gpu_put_enabled + + args = resolved_view(self.s.server_args) + parallel = get_parallel() + schedule = get_schedule() + features = get_exec().features + forbidden = { + "chunked_prefill": schedule.chunked_prefill_size != -1, + "PP": parallel.pp_size != 1, + "DP": parallel.dp_size != 1, + "attention_DP": parallel.attn_dp_size != 1, + "CP": parallel.attn_cp_size != 1 or parallel.dcp_size != 1, + "HiCache": bool(args.enable_hierarchical_cache), + "LoRA": bool(args.enable_lora), + "sessions": bool(args.enable_session_radix_cache) + or bool(getattr(args, "enable_streaming_session", False)), + "offload": any(bool(getattr(args, name, False)) for name in ( + "hicache_storage_backend", "enable_lmcache", "enable_flexkv", + "disaggregation_decode_enable_offload_kvcache")), + "GPU_publication": gpu_put_enabled(), + "priority_preemption": bool(self.s.enable_priority_preemption), + "prefill_delayer": bool(self.s.prefill_delayer) + or bool(getattr(self.s, "min_free_slots_delayer", None)), + "DLLM": self.s.dllm_config is not None, + "speculative_decode": not self.s.spec_algorithm.is_none(), + "radix_disabled": self.enabled and bool(args.disable_radix_cache), + "memory_saver": bool(args.enable_memory_saver), + "deferred_weights": bool(getattr(args, "wait_for_weights_before_ready", False)), + } + if bad := [name for name, present in forbidden.items() if present]: + raise CaptureContractError("HS capture unsupported configuration: " + ", ".join(bad)) + if self.enabled: + if not args.enable_deterministic_inference: + raise CaptureContractError("HS reuse requires --enable-deterministic-inference") + incompatible = [name for name in ( + "enable_int8_mamba_checkpoint", "enable_mamba_cache_stochastic_rounding", + "enable_linear_replayssm", "enable_linear_replayssm_spec", + ) if getattr(args, name, False)] + if incompatible: + raise CaptureContractError("HS reuse requires exact recurrent checkpoints: " + + ", ".join(incompatible)) + from sglang.srt.managers.prefix_admission import validate_capture_checkpoint_support + checkpoint = validate_capture_checkpoint_support(self.s) + else: + checkpoint = {"reuse": False} + if self.config.chunk_tokens % self.s.page_size: + raise CaptureContractError("HS chunk tokens must be a multiple of the KV page size") + layers = tuple(features.return_aux_hidden_state_layer_ids or ()) + if not layers: + raise CaptureContractError("HS capture needs ordered auxiliary layer IDs") + text = self.s.model_config.hf_text_config + width = int(text.hidden_size) + dtype = str(self.s.model_config.dtype).removeprefix("torch.") + schema = FeatureSchema((FeatureSpec("aux", width * len(layers), dtype, layers), + FeatureSpec("last_hidden", width, dtype, ()))) + manifest = { + "model": os.path.realpath(args.model_path), "revision": args.revision, + "config": text.to_dict(), "dtype": dtype, + "quantization": args.quantization, "load_format": str(args.load_format), + "weights": "immutable-for-load-epoch", "tp_size": parallel.tp_size, + "deterministic_inference": bool(args.enable_deterministic_inference), + "kv_dtype": args.kv_cache_dtype, "bf16_gemm_backend": args.bf16_gemm_backend, + "attention_backend": args.attention_backend, + "prefill_attention_backend": args.prefill_attention_backend, + "linear_attention_backend": getattr(args, "linear_attn_backend", None), + "checkpoint_semantics": checkpoint, + "capture_method": str(features.aux_hidden_state_capture), + "aux_semantics": "raw_ordered_capture", "last_hidden_semantics": "post_norm", + "layout": "contiguous_token_width", "positions": "standard_text_causal", + } + return manifest, schema + + def capability(self): + return {"protocol_version": 1, "server_epoch": self.epoch, + "begin_batch": True, "published_ownership": "attempt-v1", + "capture_namespace": self.namespace, "capture_signature": self.signature, + "schema": self.schema_wire, "cache_enabled": self.enabled, + "transport": "host", "no_oversize_fallback": True, + "support": {"nonchunked": True, "host_publication": True, + "causal_text_only": True, "pp_size": 1, "dp_size": 1, + "offload": False, "sessions": False, "lora": False, + "weight_updates": False, "complete_schema": True}, + "limits": {name: getattr(self.config, name) for name in ( + "attempt_capacity", "chunk_tokens", "cache_bytes", "pinned_bytes", + "output_bytes", "device_bytes", "max_workers")}} + + def _claim(self, req): + from specforge_sglang_capture.capture import request_spec + spec = request_spec(req) + if not spec or "protocol_version" not in spec: + return None + for name, expected in (("protocol_version", 1), ("server_epoch", self.epoch), + ("capture_namespace", self.namespace), + ("capture_signature", self.signature)): + if spec.get(name) != expected: + raise CaptureContractError("capture " + name + " mismatch") + if spec.get("object_key_layout") != "attempt-v1": + raise CaptureContractError("HS capture requires negotiated immutable attempt keys") + if req.sampling_params.max_new_tokens != 0: + raise CaptureContractError("HS capture requires max_new_tokens=0") + if any(getattr(req, name, None) is not None for name in ( + "multimodal_inputs", "session", "lora_id", "positional_embed_overrides", + "cache_salt", "session_id", "input_embeds")): + raise CaptureContractError("HS capture supports standard causal text only") + if set(spec.get("features", {})) != {"aux", "last_hidden"}: + raise CaptureContractError("HS capture requires complete aux and last_hidden schema") + names = list(spec["features"].values()) + for item in spec.get("passthrough", []): + if not isinstance(item, dict): + raise CaptureContractError("passthrough must contain objects") + names.append(item.get("name")) + shape = item.get("shape") + if not isinstance(shape, list) or any(type(d) is not int or d < 0 for d in shape): + raise CaptureContractError("invalid passthrough shape") + count = 1 + for dim in shape: + count *= dim + def elements(value): + return sum(elements(v) for v in value) if isinstance(value, list) else 1 + if elements(item.get("data")) != count: + raise CaptureContractError("passthrough data does not match its declared shape") + if any(not isinstance(name, str) or not name for name in names) or len(set(names)) != len(names): + raise CaptureContractError("capture output names must be distinct nonempty strings") + if spec.get("request_digest", digest_request(req.origin_input_ids, spec)) != digest_request(req.origin_input_ids, spec): + raise CaptureContractError("capture request digest mismatch") + entry = getattr(req, "_specforge_hs_attempt", None) + if entry is None: + entry = self.registry.claim(spec, req.origin_input_ids) + entry.request = req + req._specforge_hs_attempt = entry + elif (entry.revoked or entry.request is not req + or self.registry.lookup(spec) is not entry): + raise CaptureContractError("capture attempt is stale or revoked") + req.extra_key = self.namespace if self.enabled else "spec-attempt:" + entry.token + return entry + + def prepare(self, scheduler, req): + def membership(): + self._assert_owner() + from specforge_sglang_capture.capture import request_spec + spec = request_spec(req) + return req.rid, bool(spec and "protocol_version" in spec) + + member = self._local_phase("prepare.input", membership) + self._agree("prepare.membership", member) + if not member[1]: + return None + self._sync_transfers("prepare.transfers") + return self._local_phase("prepare.lease", lambda: self._prepare_local(req)) + + def _prepare_local(self, req): + if self._closed or self.fatal_capture_error: + raise CaptureContractError(self.fatal_capture_error or "HS runtime is closed") + entry = self._claim(req) + execution = ExecutionIdentity(self.epoch, req.rid, entry.attempt_id, entry.execution_epoch) + if req.rid in self.candidates: + raise CaptureContractError("HS admission candidate already active") + candidate = _Candidate(execution, entry, tuple(req.origin_input_ids)) + self.candidates[req.rid] = candidate + req._specforge_hs_execution = execution + if not candidate.tokens: + raise CaptureContractError("HS capture requires a nonempty prompt") + cap = 0 + if self.writer: + candidate.lease = self.cache.lookup_and_pin( + self.namespace, candidate.tokens, + len(candidate.tokens) - 1 if self.enabled and not getattr(req, "_specforge_hs_force_full", False) else 0) + cap = candidate.lease.coverage_end + from sglang.srt.managers.prefix_admission import PrefixAdmissionProposal + return PrefixAdmissionProposal(cap, dataclasses.astuple(execution), candidate) + + def after_match(self, scheduler, req, proposal, lawful_prefix_len): + self._assert_owner() + candidate = self.candidates[req.rid] + if not 0 <= lawful_prefix_len <= len(candidate.tokens) - 1: + raise CaptureContractError("native recovery prefix is outside [0,N-1]") + if self.writer: + candidate.lease = self.cache.shrink_lease(candidate.lease, lawful_prefix_len) + candidate.L = lawful_prefix_len + + def reserve(self, scheduler, req, proposal, lawful_prefix_len): + self._assert_owner() + candidate = self.candidates[req.rid] + if candidate.L != lawful_prefix_len: + raise CaptureContractError("native recovery prefix changed after match") + if not self.writer: + return True + from specforge_sglang_capture.capture import request_spec + passthrough = 0 + from specforge_sglang_capture.sink import STR_DTYPE + for item in request_spec(req).get("passthrough", []): + dtype = STR_DTYPE.get(item.get("dtype", "int64")) + if dtype is None: + raise CaptureContractError("unsupported passthrough dtype") + shape = item.get("shape") + if not isinstance(shape, list) or any(type(d) is not int or d < 0 for d in shape): + raise CaptureContractError("invalid passthrough shape") + elements = 1 + for dim in shape: + elements *= dim + passthrough += elements * torch.empty((), dtype=dtype).element_size() + candidate.reservation = self.controller.try_reserve_output( + candidate.execution, len(candidate.tokens), lawful_prefix_len, passthrough) + return candidate.reservation is not None + + def commit(self, scheduler, req, proposal, lawful_prefix_len): + self._assert_owner() + candidate = self.candidates[req.rid] + if candidate.L != lawful_prefix_len: + raise CaptureContractError("native recovery prefix changed before commit") + if self.writer: + req._specforge_hs_plan = self.controller.freeze_plan( + candidate.execution, self.namespace, candidate.tokens, len(candidate.tokens), + lawful_prefix_len, candidate.lease, candidate.reservation) + candidate.entry.state = "ADMITTED" + candidate.entry.last_plan = {"N": len(candidate.tokens), "L": lawful_prefix_len, + "H": proposal.max_prefix_len, "span": [lawful_prefix_len, len(candidate.tokens)], + "execution_epoch": candidate.execution.execution_epoch, "rank": self.tp_rank} + self.history.append({"attempt_id": candidate.entry.attempt_id, + **candidate.entry.last_plan}) + + def rollback(self, scheduler, req, proposal, reason): + self._assert_owner() + candidate = self.candidates.pop(req.rid, None) + if candidate is None: + return + if self.writer: + if candidate.reservation is not None: + self.controller.release_output(candidate.reservation) + if candidate.lease is not None: + self.cache.release_lease(candidate.lease) + self.controller.poll_completed() + self.controller.poll_drained() + req._specforge_hs_plan = None + candidate.entry.execution_epoch += 1 + req._specforge_hs_execution = None + candidate.entry.state = "QUEUED" + + def lifecycle(self, scheduler, req, event, identity): + self._assert_owner() + if event == "shutdown": + self.shutdown() + return + if event == "weight_update": + raise CaptureContractError("HS capture forbids weight updates within a load epoch") + if req is None: + return + entry = getattr(req, "_specforge_hs_attempt", None) + if entry is None: + from specforge_sglang_capture.capture import request_spec + spec = request_spec(req) + if not spec or "protocol_version" not in spec: + return + try: + entry = self.registry.lookup(spec) + except (KeyError, ValueError): + return + entry.request = req + req._specforge_hs_attempt = entry + current = getattr(req, "_specforge_hs_execution", None) + if identity is not None and identity not in (current, dataclasses.astuple(current) if current else None): + return + if event == "finish": + return # Native finish precedes consume/deferred publication. + if entry.transfer_certificate: + return # Receiver already owns the immutable published objects. + candidate = self.candidates.pop(req.rid, None) + if candidate is not None and self.writer: + if candidate.reservation is not None: + entry.drains.append(self.controller.invalidate(candidate.execution, event)) + if candidate.lease is not None: + self.cache.release_lease(candidate.lease) + for job in tuple(self.jobs.values()): + if job.entry is entry and (current is None or job.execution == current): + self._cancel_job(job, event, revoke=event not in ("retract", "retry")) + req._specforge_hs_plan = None + req._specforge_hs_execution = None + entry.execution_epoch += 1 + if event in ("retract", "retry"): + req._specforge_hs_force_full = True + entry.state = "QUEUED" + else: + entry.revoked, entry.state = True, "DRAINING" + self._mark_drained(entry) + + def invalidate(self, entry, reason): + if entry.transfer_certificate: + return + entry.revoked, entry.state = True, "DRAINING" + if entry.request is not None: + self.lifecycle(self.s, entry.request, reason, + getattr(entry.request, "_specforge_hs_execution", None)) + self._mark_drained(entry) + + def _mark_drained(self, entry): + if (entry.state != "RETIRED" and entry.revoked and entry.deliveries == 0 + and all(handle.done() for handle in entry.drains)): + entry.state = "WRITER_DRAINED" + + def _current(self, job): + return ((not job.entry.revoked or bool(job.entry.transfer_certificate)) + and self._same_execution(job)) + + def _same_execution(self, job): + return (job.entry.execution_epoch == job.execution.execution_epoch + and job.entry.attempt_id == job.execution.attempt_id) + + def _cancel_job(self, job, reason, *, revoke=True): + if job.entry.transfer_certificate: + return + if revoke: + job.entry.revoked, job.entry.state = True, "DRAINING" + if job.failed is None: + job.failed = RuntimeError(reason) + job.entry.drains.append(self.controller.invalidate(job.execution, reason)) + + def snapshot(self, req, item, aux, last_hidden): + """Freeze compact owned suffixes on the forward stream exactly once.""" + self._assert_owner() + plan = req._specforge_hs_plan + candidate = self.candidates.pop(req.rid) + if plan.execution != candidate.execution or plan.execution in self.jobs: + raise CaptureContractError("capture plan execution is stale or already submitted") + features = {} + try: + for name, tensor in (("aux", aux), ("last_hidden", last_hidden)): + if tensor is None: + raise CaptureContractError("capture is missing " + name) + features[name] = tensor[item.start:item.start + item.length].detach().clone( + memory_format=torch.contiguous_format) + ready = None + if next(iter(features.values())).is_cuda: + ready = torch.cuda.Event() + ready.record() + future = self.controller.submit(plan, CapturedSpan( + plan.execution, plan.L, plan.N, features, ready)) + except BaseException as exc: + try: + if any(t.is_cuda for t in features.values()): + torch.cuda.current_stream(next(iter(features.values())).device).synchronize() + except BaseException as drain_error: + retained = Future() + candidate.entry.drains.append(retained) + self.quarantined_snapshots.append((plan, features, retained)) + self.fatal_capture_error = "capture snapshot drain unknown: " + str(drain_error) + raise CaptureContractError(self.fatal_capture_error) from None + candidate.entry.drains.append(self.controller.invalidate(plan.execution, "snapshot-failed")) + self.invalidate(candidate.entry, "snapshot-failed") + raise exc + output = Future() + output.set_running_or_notify_cancel() + from specforge_sglang_capture.capture import request_spec + job = _Publication(plan.execution, candidate.entry, copy.deepcopy(request_spec(req)), + plan, future, output) + if self._diagnostics: + sequence = self._diagnostic_sequence + self._diagnostic_sequence += 1 + if sequence % self._diagnostic_every == 0 and sequence // self._diagnostic_every < self._diagnostic_limit: + job.diagnostic = {"p0_v1_event": "publication", "server_epoch": self.epoch, + "request_id": plan.execution.request_id, "attempt_id": plan.execution.attempt_id, + "execution_epoch": plan.execution.execution_epoch, "rank": self.tp_rank, + "store_id": job.spec["store_id"], "sample_id": job.spec["sample_id"], + "request_digest": job.entry.request_digest, "N": plan.N, "L": plan.L, + "suffix_tokens": plan.N - plan.L, "p0_v1_snapshot_ns": time.monotonic_ns()} + self.jobs[plan.execution] = job + candidate.entry.deliveries += 1 + return job + + def consume(self, jobs, batch): + self._assert_owner() + reqs = {req.rid: req for req in batch.reqs} + if self._diagnostics: + self._diagnostic_batch += 1 + for job in jobs: + if job.diagnostic is not None: + # One ID per native consume batch; do not sum shared batch work + # once per sample. This timestamp is NOT a CUDA forward timer. + job.diagnostic.update(p0_v1_consume_batch_id=self._diagnostic_batch, + p0_v1_consume_batch_samples=len(jobs), + p0_v1_consume_ns=time.monotonic_ns()) + req = reqs.get(job.execution.request_id) + if req is None or not self._current(job): + self._cancel_job(job, "capture request disappeared before consume") + continue + if not req.finished(): + self._cancel_job(job, "capture requests must finish at prefill") + self.capture.results[req.rid] = {"sample_id": job.spec["sample_id"], + "error": "capture requests must finish at prefill"} + continue + job.consumed = True + self.capture.defer_response(req, HSCompletion(self, job)) + self.poll() + + def capture_error(self, req, reason): + self._assert_owner() + entry = getattr(req, "_specforge_hs_attempt", None) + if entry is not None: + self.invalidate(entry, reason) + + def _drop_job(self, job): + if self.jobs.get(job.execution) is not job: + return + self.jobs.pop(job.execution) + job.entry.deliveries -= 1 + if job.entry.deliveries < 0: + raise RuntimeError("capture delivery ownership underflow") + self._mark_drained(job.entry) + + def _finish_error(self, job, error): + if self._current(job): + self._cancel_job(job, str(error)) + if job.sample is not None: + job.sample.release() + job.sample = None + if not job.output.done(): + job.output.set_exception(scalar_error(error)) + self._drop_job(job) + + def _validate_publication(self, job, result): + if not isinstance(result, dict): + raise CaptureContractError("capture sink result is not an object") + for name in ("store_id", "sample_id", "gen"): + if result.get(name) != job.spec[name]: + raise CaptureContractError("capture sink changed " + name) + names = [*job.spec["features"].values(), + *(item["name"] for item in job.spec.get("passthrough", []))] + base = (f"{job.spec['store_id']}/{job.spec['sample_id']}/g1/attempt/" + f"{job.execution.server_epoch}/{job.spec['reservation_token']}") + keys = {name: base + "/" + name for name in names} + if set(result.get("features", {})) != set(keys): + raise CaptureContractError("capture sink returned incomplete feature names") + if any(result["features"][name].get("key") != key for name, key in keys.items()): + raise CaptureContractError("capture sink immutable physical keys mismatch") + for feature in self.schema.features: + name = job.spec["features"][feature.name] + meta = result["features"][name] + if meta.get("shape") != [1, job.plan.N, feature.width] or meta.get("dtype") != feature.dtype: + raise CaptureContractError("capture sink feature schema mismatch") + for item in job.spec.get("passthrough", []): + meta = result["features"][item["name"]] + if meta.get("shape") != item["shape"] or meta.get("dtype") != item.get("dtype", "int64"): + raise CaptureContractError("capture sink passthrough schema mismatch") + if result.get("aux_layer_ids") != list(self.schema.features[0].layer_ids): + raise CaptureContractError("capture sink auxiliary layer order mismatch") + return keys + + def poll(self, scheduler=None): + """Progress independent jobs and batch only currently ready publications.""" + self._assert_owner() + if not self.writer: + return + self.controller.poll_completed() + self.controller.poll_drained() + + ready = [] + for job in tuple(self.jobs.values()): + if job.cleanup is not None: + if not job.cleanup.done(): + continue + try: + job.cleanup.result() + except Exception as exc: + # Writer has stopped reading, but deletion remains unproven. + # Keep this exact attempt live so the client can retry cleanup. + logger.error("capture cleanup failed for %s: %s", job.execution, exc) + self._finish_error(job, job.failed or RuntimeError("capture cancelled")) + continue + if job.sink is not None or not job.materialize.done(): + continue + try: + if job.sample is None: + job.sample = job.materialize.result() + if job.failed is not None or not self._current(job): + raise job.failed or RuntimeError("capture execution invalidated") + if self.fatal_capture_error: + raise RuntimeError(self.fatal_capture_error) + if job.consumed: + ready.append(job) + except Exception as exc: + self._finish_error(job, exc) + if ready: + samples = [(job.spec, job.sample.features["aux"], + job.sample.features["last_hidden"]) for job in ready] + enqueue_ns = time.monotonic_ns() if self._diagnostics else 0 + try: + future = self.capture.sink.submit_samples(samples) + except Exception as exc: + samples.clear() + future = Future() + future.set_exception(scalar_error(exc)) + for index, job in enumerate(ready): + job.sink, job.sink_index, job.sink_count = future, index, len(ready) + job.publish_started = time.perf_counter() + if job.diagnostic is not None: + job.diagnostic["p0_v1_runtime_sink_enqueue_ns"] = enqueue_ns + done_ns = job.sample.timings.get("p0_v1_materialize_done_ns") + if done_ns is not None: + job.diagnostic["p0_v1_materialize_to_enqueue_ms"] = (enqueue_ns - done_ns) / 1e6 + job.entry.state = "PUBLISHING" + samples = None # The sink now owns and clears its argument list. + ready.clear() + for job in tuple(self.jobs.values()): + if job.sink is None or not job.sink.done() or job.cleanup is not None: + continue + observed_ns = time.monotonic_ns() if job.diagnostic is not None else 0 + try: + results = job.sink.result() + if len(results) != job.sink_count: + raise CaptureContractError("capture sink batch result count mismatch") + result = results[job.sink_index] + keys = self._validate_publication(job, result) + if job.failed is not None or not self._current(job): + job.failed = job.failed or RuntimeError("capture cancelled during publication") + # Keep the publication owner until the exact-key deletion is + # ordered after its already-completed writer. + job.cleanup = self.capture.sink.discard_sample(job.spec) + continue + entry = job.entry + if entry.deliveries != 1 or entry.readers or any(not handle.done() for handle in entry.drains): + raise CaptureContractError("capture transfer precedes all execution drains") + timings = dict(job.sample.timings) + timings["publication_queue_ms"] = (time.perf_counter() - job.publish_started) * 1000 + if job.diagnostic is not None: + observed = observed_ns + sink_timing = dict(getattr(job.sink, "p0_v1_timings", {})) + job.diagnostic.update(timings) + job.diagnostic.update(sink_timing) + job.diagnostic["p0_v1_owner_observed_ns"] = observed + if "p0_v1_sink_done_ns" in sink_timing: + job.diagnostic["p0_v1_sink_done_to_owner_ms"] = (observed - sink_timing["p0_v1_sink_done_ns"]) / 1e6 + job.diagnostic["p0_v1_sink_queue_ms"] = (sink_timing["p0_v1_sink_start_ns"] - sink_timing["p0_v1_sink_enqueue_ns"]) / 1e6 + timings.update({key: value for key, value in job.diagnostic.items() + if key.startswith("p0_v1_")}) + self.diagnostics.append(job.diagnostic) + entry.last_plan["timings"] = timings + # Sink has cleared its source refs. Transfer allocation ownership + # to cache (or retire it), then issue the physical-key certificate. + job.sample.release() + job.sample = None + self._drop_job(job) + entry.state = "PUBLISHED" + certificate = {**self._wire(entry), "publication_ownership": "attempt-v1", + "object_key_layout": "attempt-v1", "physical_keys": keys, + "schema": self.schema_wire} + entry.transfer_certificate = {"publication_ownership": "attempt-v1", + "physical_keys": keys, "publication_certificate": certificate} + result.update(self._wire(entry)) + result.update(capture_namespace=self.namespace, capture_signature=self.signature, + reused_tokens=job.plan.L, schema=self.schema_wire) + self.registry.retire(entry) + self._transferred_tokens.append(entry.token) + if not job.output.done(): + job.output.set_result(result) + except Exception as exc: + if getattr(exc, "capture_writer_drained", True) is False: + self._cancel_job(job, "native publication drain unknown") + unknown = Future() + job.entry.drains.append(unknown) + self.quarantined_publications.append(job) + self.fatal_capture_error = "capture publication drain unknown; buffers and budgets retained" + if not job.output.done(): + job.output.set_exception(scalar_error(exc)) + self._drop_job(job) + else: + self._finish_error(job, exc) + for entry in self.registry.active.values(): + self._mark_drained(entry) + self.controller.poll_completed() + self.controller.poll_drained() + + def has_pending_work(self): + # Writer-local drain progress, like native deferred outputs. This must + # not be used as an unsynchronized TP collective idle predicate. + return bool(self.jobs or self.quarantined_snapshots or self.quarantined_publications) + + def shutdown(self): + self._assert_owner() + if self._closed: + return + self._closed = True + for entry in list(self.registry.active.values()): + self.invalidate(entry, "server-shutdown") + while self.jobs: + self.poll() + if self.jobs: + time.sleep(0.001) + if self.quarantined_snapshots or self.quarantined_publications: + raise RuntimeError("HS shutdown retains quarantined storage; native drain is unproven") + if self.controller: + self.controller.shutdown(wait=True) + self.capture.sink.close() + + def _wire(self, entry): + return {"protocol_version": 1, "server_epoch": self.epoch, "store_id": entry.identity[0], + "sample_id": entry.identity[1], "gen": 1, "attempt_id": entry.attempt_id, + "reservation_token": entry.token, "issue_sequence": entry.sequence, + "request_digest": entry.request_digest, + "cleanup_owner": entry.cleanup_owner, "cleanup_token": entry.cleanup_token, + "active_readers": len(entry.readers), + "state": entry.state, "writer_drained": entry.state in ("WRITER_DRAINED", "PUBLISHED", "RETIRED"), + **entry.last_plan, + **({"object_key_layout": entry.object_key_layout} if entry.object_key_layout else {}), + **entry.transfer_certificate} + + def control(self, scheduler, body): + try: + return self._control(body) + except ValueError as exc: + raise CaptureControlRejected(str(exc)) from None + + def _sync_transfers(self, phase): + def packet(): + return list(self._transferred_tokens), self.fatal_capture_error + + update = self._writer_broadcast(phase + ".payload", packet) + + def apply(): + transferred, fatal = update + if fatal: + self.fatal_capture_error = fatal + if not self.writer: + for token in transferred: + entry = self.registry.tokens.get(token) + if entry is not None: + if entry.request is not None: + self.candidates.pop(entry.request.rid, None) + entry.state = "PUBLISHED" + self.registry.retire(entry) + + # A peer retirement can fail after local bookkeeping or resource checks. + # Do not let its caller proceed to a later broadcast without this vote. + self._local_phase(phase + ".apply", apply) + self._transferred_tokens = [] + + def _control_input(self, body): + self._assert_owner() + if not isinstance(body, dict) or int(body.get("protocol_version", -1)) != 1: + raise ValueError("unsupported capture protocol") + action = body.get("action") + if action not in ("capability", "stats", "begin", "begin_batch", "resolve", + "read_acquire", "read_release", "cleanup_acquire", "cleanup_done"): + raise ValueError("unknown capture control action") + if action != "capability" and body.get("server_epoch") != self.epoch: + raise ValueError("capture server epoch changed") + # HTTP and typed IPC carry JSON. Agree the complete request, including + # action and fencing tokens, before choosing any nested control branch. + return action, json.dumps(body, sort_keys=True, separators=(",", ":")) + + def _control(self, body): + request = self._local_phase("control.input", lambda: self._control_input(body)) + self._agree("control.request", request) + self._local_phase("control.poll", self.poll) + self._sync_transfers("control.transfers") + action = request[0] + if action == "capability": + return self._local_phase("control.capability", self.capability) + if action == "stats": + def stats(): + return {**self.capability(), "active_attempts": len(self.registry.active), + "pending_deliveries": len(self.jobs), + "active_readers": sum(len(x.readers) for x in self.registry.active.values()), + "plans": list(self.history), + **({"p0_v1_diagnostics": list(self.diagnostics), + "p0_v1_diagnostic_seen": self._diagnostic_sequence, + "p0_v1_diagnostic_every": self._diagnostic_every, + "p0_v1_diagnostic_limit": self._diagnostic_limit} if self._diagnostics else {}), + "cache": self.cache.stats() if self.cache else {}, + "controller": self.controller.stats() if self.controller else {}} + return self._local_phase("control.stats", stats) + if action in ("begin", "begin_batch"): + return self._begin_control(body, action) + + kind, entry = self._local_phase("control.lookup", lambda: self._lookup_control(body)) + key = self._local_phase("control.target-key", lambda: ( + kind, entry.identity, entry.attempt_id, entry.request_digest, entry.token, entry.sequence)) + self._agree("control.target", key) + if kind == "retired": + return self._local_phase("control.retired", lambda: self._retired_control(body, entry)) + + if action == "resolve": + def resolve(): + if body.get("cancel"): + self.invalidate(entry, "external-cancel") + if entry.request is not None: + from sglang.srt.managers.io_struct import AbortReq + self.s.abort_request(AbortReq(rid=entry.request.rid)) + self._mark_drained(entry) + self._local_phase("control.resolve.local", resolve) + return self._broadcast_wire("control.resolve.wire", entry) + if action == "read_acquire": + wire = self._broadcast_wire("control.read-acquire.wire", entry) + + def validate_read(): + owner = str(body.get("reader_owner") or "") + if not owner: + raise ValueError("read owner is required") + existing = entry.readers.get(owner) + # Lost replies can recover an existing permit after revoke; + # they cannot create a new reader after its first release. + if existing is None: + if any(old == owner for old, _ in entry.released_readers): + raise ValueError("released read owner cannot acquire again") + if len(entry.readers) + len(entry.released_readers) >= self.registry.capacity: + raise ValueError("attempt read owner history capacity exhausted") + if entry.revoked or wire["state"] != "PUBLISHED" or entry.cleanup_token is not None: + raise ValueError("attempt is not readable") + if sum(len(x.readers) for x in self.registry.active.values()) >= self.registry.capacity: + raise ValueError("read permit capacity exhausted") + return owner, existing + + owner, existing = self._local_phase("control.read-acquire.validate", validate_read) + self._agree("control.read-acquire.branch", (owner, existing)) + token = existing + if token is None: + token = self._writer_broadcast("control.read-acquire.token", lambda: uuid.uuid4().hex) + + def acquire_read(): + entry.readers[owner] = token + return dict(self._wire(entry), reader_owner=owner, reader_token=token) + return self._local_phase("control.read-acquire.commit", acquire_read) + if action == "read_release": + def release_read(): + owner, token = body.get("reader_owner"), body.get("reader_token") + if entry.readers.get(owner) == token and token is not None: + entry.readers.pop(owner) + entry.released_readers.append((owner, token)) + elif (owner, token) not in entry.released_readers: + raise ValueError("unknown read permit") + return dict(self._wire(entry), reader_owner=owner, reader_token=token, reader_released=True) + return self._local_phase("control.read-release", release_read) + if action == "cleanup_acquire": + self._local_phase("control.cleanup-acquire.drain", lambda: self._mark_drained(entry)) + wire = self._broadcast_wire("control.cleanup-acquire.wire", entry) + + def validate_cleanup(): + owner = str(body.get("cleanup_owner") or "") + if not owner or not wire["writer_drained"] or entry.readers: + raise ValueError("writer/readers not drained or cleanup owner absent") + if entry.cleanup_owner is not None and entry.cleanup_owner != owner: + raise ValueError("another store owns sample cleanup") + return owner, entry.cleanup_token + + owner, existing = self._local_phase("control.cleanup-acquire.validate", validate_cleanup) + self._agree("control.cleanup-acquire.branch", (owner, existing)) + token = existing + if token is None: + token = self._writer_broadcast("control.cleanup-acquire.token", lambda: uuid.uuid4().hex) + + def acquire_cleanup(): + entry.cleanup_owner, entry.cleanup_token = owner, token + return self._wire(entry) + return self._local_phase("control.cleanup-acquire.commit", acquire_cleanup) + if action == "cleanup_done": + def validate_done(): + if entry.cleanup_token is None or body.get("cleanup_token") != entry.cleanup_token: + raise ValueError("cleanup permit mismatch") + self._mark_drained(entry) + self._local_phase("control.cleanup-done.validate", validate_done) + self._broadcast_wire("control.cleanup-done.wire", entry) + + def retire(): + self.registry.retire(entry) + return self._wire(entry) + return self._local_phase("control.cleanup-done.retire", retire) + raise AssertionError("validated control action has no implementation") + + def _begin_control(self, body, action): + def validate(): + if self.fatal_capture_error: + raise ValueError(self.fatal_capture_error) + bodies = body.get("tasks") if action == "begin_batch" else [body] + if not isinstance(bodies, list) or not bodies or len(bodies) > self.registry.capacity: + raise ValueError("invalid begin_batch size") + if any(not isinstance(item, dict) or item.get("server_epoch") != self.epoch for item in bodies): + raise ValueError("capture server epoch changed") + return bodies + + bodies = self._local_phase("control.begin.validate", validate) + tokens = self._writer_broadcast("control.begin.tokens", lambda: [uuid.uuid4().hex for _ in bodies]) + prepared = self._local_phase("control.begin.preflight", lambda: self._begin_trial(bodies, tokens)) + + def proposal(): + if prepared is None: + return "capacity-declined", self.registry.capacity + _, entries = prepared + return "ready", tuple((entry.identity, entry.attempt_id, entry.request_digest, + entry.token, entry.sequence, entry.object_key_layout) for entry in entries) + + decision = self._local_phase("control.begin.proposal", proposal) + self._agree("control.begin.decision", decision) + if prepared is None: + # Every rank declined an uncommitted trial. No actual registry has + # granted anything in this operation, so this is the ONLY typed + # capacity refusal. Divergent preflights or commit failures are + # generic fail-closed errors, never proof of an ungranted mutation. + return {"error": "attempt registry capacity exhausted", + "error_code": "capture_capacity_declined", "reservation_granted": False} + trial, entries = prepared + replies = self._local_phase("control.begin.reply", lambda: [self._wire(entry) for entry in entries]) + + def commit(): + self.registry.active, self.registry.tokens, self.registry.sequence = ( + trial.active, trial.tokens, trial.sequence) + self._local_phase("control.begin.commit", commit) + return {"server_epoch": self.epoch, "attempts": replies} if action == "begin_batch" else replies[0] + + def _begin_trial(self, bodies, tokens): + trial = AttemptRegistry(self.registry.epoch, self.registry.capacity) + trial.active, trial.tokens, trial.sequence = ( + dict(self.registry.active), dict(self.registry.tokens), self.registry.sequence) + try: + entries = trial.begin_batch(bodies, tokens) + except AttemptCapacityDeclined: + return None + return trial, entries + + def _lookup_control(self, body): + try: + return "active", self.registry.lookup(body) + except ValueError: + identity = self.registry.identity(body) + entry = next((entry for entry in self.registry.retired + if entry.token == body.get("reservation_token") and entry.identity == identity + and entry.attempt_id == body.get("attempt_id")), None) + if entry is None: + raise + return "retired", entry + + def _retired_control(self, body, entry): + action = body["action"] + if action == "read_release": + pair = body.get("reader_owner"), body.get("reader_token") + if pair not in entry.released_readers: + raise ValueError("unknown retired read permit") + return dict(self._wire(entry), reader_owner=pair[0], reader_token=pair[1], reader_released=True) + if action in ("cleanup_acquire", "cleanup_done", "resolve"): + if action == "cleanup_done" and body.get("cleanup_token") != entry.cleanup_token: + raise ValueError("cleanup permit mismatch") + return self._wire(entry) + raise ValueError("unknown, retired or mismatched reservation token") diff --git a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py index 269441e40c..ec23ca57fe 100644 --- a/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py +++ b/plugins/sglang-spec-capture/specforge_sglang_capture/sink.py @@ -111,6 +111,25 @@ def object_key(store_id: str, sample_id: str, gen: int, name: str) -> str: return f"{store_id}/{sample_id}/g{gen}/{name}" +def sample_object_key(spec: Dict[str, Any], name: str) -> str: + if spec.get("object_key_layout") == "attempt-v1": + if not spec.get("server_epoch") or not spec.get("reservation_token"): + raise ValueError("immutable capture keys require epoch and reservation token") + return (f"{spec['store_id']}/{spec['sample_id']}/g{int(spec.get('gen', 1))}" + f"/attempt/{spec['server_epoch']}/{spec['reservation_token']}/{name}") + return object_key(str(spec["store_id"]), str(spec["sample_id"]), + int(spec.get("gen", 1)), name) + + +class CaptureWriterUncertain(RuntimeError): + """A native write may still be reading its registered source allocation.""" + capture_writer_drained = False + + +class _DrainedWriteFailure(RuntimeError): + """A complete status vector proves the native call has returned ownership.""" + + class _CudaBytes: """Exposes a raw device allocation to ``torch.as_tensor`` without copying.""" @@ -303,10 +322,20 @@ class SpecCaptureSink: def __init__(self, aux_layer_ids: Optional[Sequence[int]] = None) -> None: self.aux_layer_ids = list(aux_layer_ids) if aux_layer_ids else None self._store = None + self._owns_store = False self._put_config = None self._nvlink: Optional[NvlinkArena] = None self._control_listener: Optional[socket.socket] = None self._lock = threading.Lock() + self._uncertain_publications = [] + self._uncertain_lock = threading.Lock() + self._closed = False + self.require_hard_pin = False + self._diagnostics = os.environ.get("SPECFORGE_HS_DIAGNOSTICS", "0") == "1" + self._diagnostic_every = max(1, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_EVERY", "1"))) + self._diagnostic_limit = max(0, int(os.environ.get("SPECFORGE_HS_DIAGNOSTICS_MAX", "4096"))) + self._diagnostic_sequence = 0 + self._diagnostic_local = threading.local() # One writer is enough: Mooncake stripes a batched transfer itself. # The thread only decouples that transfer from the scheduler. self._executor = ThreadPoolExecutor( @@ -373,10 +402,13 @@ def _connect(self): # Older ROCm Mooncake builds only expose `with_soft_pin`. if hasattr(cfg, "with_hard_pin"): cfg.with_hard_pin = True + elif self.require_hard_pin: + raise RuntimeError("HS publication requires Mooncake hard-pin ownership") elif hasattr(cfg, "with_soft_pin"): cfg.with_soft_pin = True self._put_config = cfg self._store = store + self._owns_store = True logger.info("spec-capture mooncake sink connected") return store @@ -429,12 +461,7 @@ def discard_sample(self, spec: Dict[str, Any]) -> Future: names.extend(item["name"] for item in spec.get("passthrough") or []) keys = list( dict.fromkeys( - object_key( - str(spec["store_id"]), - str(spec["sample_id"]), - int(spec.get("gen", 1)), - str(name), - ) + sample_object_key(spec, str(name)) for name in names ) ) @@ -449,7 +476,17 @@ def report_failure(done): future.add_done_callback(report_failure) return future + def _uncertain_snapshot(self): + with self._uncertain_lock: + return tuple(self._uncertain_publications) + + def _retain_uncertain(self, item): + with self._uncertain_lock: + self._uncertain_publications.append(item) + def _discard_keys(self, keys: List[str]) -> None: + if any(set(keys).intersection(item["keys"]) for item in self._uncertain_snapshot()): + raise CaptureWriterUncertain("cannot remove objects before native writer drain") if self._nvlink is not None: self._nvlink.free(keys) return @@ -486,12 +523,73 @@ def submit_samples( ``ready_event`` orders device tensors after the work that produced them. """ - return self._executor.submit(self._put_ready_samples, samples, ready_event) + if self._closed: + raise RuntimeError("capture sink is closed") + # Own a separate list: the caller may still be assigning completions + # while this worker finishes and drops its last source references. + diagnostic = None + if self._diagnostics: + with self._lock: + sequence = self._diagnostic_sequence + self._diagnostic_sequence += 1 + if sequence % self._diagnostic_every == 0 and sequence // self._diagnostic_every < self._diagnostic_limit: + diagnostic = {"p0_v1_sink_batch_id": sequence, + "p0_v1_sink_batch_samples": len(samples), + "p0_v1_sink_enqueue_ns": time.monotonic_ns()} + owned = list(samples) + # submit() may enqueue and then fail while starting a worker. Keep a + # separate immutable owner: the accepted worker clears its own list. + retained = tuple(owned) + keys = tuple(sample_object_key(spec, str(name)) + for spec, _, _ in retained + for name in [*(spec.get("features") or {}).values(), + *(item["name"] for item in spec.get("passthrough") or [])]) + try: + future = self._executor.submit(self._put_ready_samples, owned, ready_event, diagnostic) + except BaseException as exc: + self._retain_uncertain({"samples": retained, "keys": keys, + "registered": (), "acceptance_unknown": True}) + raise CaptureWriterUncertain("capture enqueue acceptance unknown: " + str(exc)) from None + # Scalar-only metadata: never retain another tensor/owner reference. + # The worker finishes writing it before the Future becomes done. + if diagnostic is not None: + future.p0_v1_timings = diagnostic + return future - def _put_ready_samples(self, samples, ready_event): - if ready_event is not None: - ready_event.synchronize() - return self.put_samples(samples) + def _put_ready_samples(self, samples, ready_event, diagnostic=None): + if diagnostic is not None: + self._diagnostic_local.current = diagnostic + diagnostic["p0_v1_sink_start_ns"] = time.monotonic_ns() + error = None + try: + if ready_event is not None: + ready_event.synchronize() + result = self.put_samples(samples) + except BaseException as exc: + # A Future retains its exception traceback. It must not retain + # drained sink tensor aliases across cache adoption. + from specforge_sglang_capture.hs.attempts import scalar_error + error = scalar_error(exc) + finally: + samples.clear() + if diagnostic is not None: + diagnostic["p0_v1_sink_done_ns"] = time.monotonic_ns() + self._diagnostic_local.current = None + if error is not None: + raise error + return result + + def close(self) -> None: + self._closed = True + self._executor.shutdown(wait=True, cancel_futures=False) + if self._uncertain_snapshot(): + raise RuntimeError("capture sink retains native writes whose drain is unproven") + if self._control_listener is not None and self._nvlink is None: + self._control_listener.close() + if self._owns_store and self._store is not None: + close = getattr(self._store, "close", None) + if close is not None: + close() def put_samples(self, samples: List[Sample]) -> List[Dict[str, Any]]: """Publish samples with one batched Mooncake call; return their results. @@ -501,6 +599,8 @@ def put_samples(self, samples: List[Sample]) -> List[Dict[str, Any]]: """ if not samples: return [] + if self._uncertain_snapshot(): + raise CaptureWriterUncertain("capture native writer is quarantined") store = None if nvlink_enabled() else self._connect() timing_enabled = os.environ.get("SGLANG_SPEC_CAPTURE_TIMING", "0") == "1" @@ -518,11 +618,11 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): tensor = tensor.detach().contiguous() else: tensor = tensor.detach().to("cpu").contiguous() - key = object_key(store_id, sample_id, gen, name) + key = sample_object_key(spec, name) keys.append(key) tensors.append(tensor) sizes.append(tensor.element_size() * tensor.numel()) - if replace: + if replace and spec.get("object_key_layout") != "attempt-v1": replace_keys.append(key) result_feats[name] = { "shape": list(tensor.shape), @@ -530,6 +630,8 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): tensor.dtype, str(tensor.dtype).replace("torch.", "") ), } + if spec.get("object_key_layout") == "attempt-v1": + result_feats[name]["key"] = key feature_meta.append(result_feats[name]) for spec, aux, last_hidden in samples: @@ -598,8 +700,14 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): ) self._remove_many_quiet(replace_keys) registered: List[int] = [] + registered_bytes = 0 seen_storage = set() + submitted = quarantined = False register_started = time.perf_counter() + diagnostic = getattr(self._diagnostic_local, "current", None) + if diagnostic is not None: + diagnostic.update(p0_v1_sink_objects=len(keys), p0_v1_sink_bytes=sum(sizes), + p0_v1_sink_staging_ms=materialize_ms) try: for tensor, nbytes in zip(tensors, sizes): storage = tensor.untyped_storage() @@ -616,6 +724,8 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): else: if rc is None or int(rc) == 0: registered.append(ptr) + if diagnostic is not None: + registered_bytes += storage.nbytes() elif tensor.is_cuda: raise RuntimeError( f"GPU capture buffer registration failed ({rc}); " @@ -623,8 +733,12 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): "PyTorch expandable_segments for this server" ) register_ms = (time.perf_counter() - register_started) * 1000.0 + if diagnostic is not None: + diagnostic["p0_v1_sink_register_ms"] = register_ms + diagnostic["p0_v1_sink_registered_bytes"] = registered_bytes put_started = time.perf_counter() batch_put = getattr(store, "batch_put_from", None) + submitted = True if batch_put is None: statuses = [ store.put_from(key, tensor.data_ptr(), nbytes, self._put_config) @@ -638,35 +752,40 @@ def stage(result_feats, *, store_id, sample_id, gen, replace, name, tensor): self._put_config, ) put_ms = (time.perf_counter() - put_started) * 1000.0 - except Exception: + if diagnostic is not None: + diagnostic["p0_v1_sink_put_ms"] = put_ms + if statuses is None: + statuses = [0] * len(keys) + if len(statuses) != len(keys): + raise CaptureWriterUncertain( + f"capture sink returned {len(statuses)} statuses for {len(keys)} objects") + if any(status is not None and int(status) == -800 for status in statuses): + raise CaptureWriterUncertain("native capture transfer drain unknown (status -800)") + failed = [(key, status) for key, status in zip(keys, statuses) + if status is not None and int(status) < 0] + if failed: + raise _DrainedWriteFailure( + f"spec-capture batch_put_from failed for {len(failed)}/{len(keys)} keys; first={failed[0]}") + except Exception as exc: + if (getattr(exc, "capture_writer_drained", True) is False + or (submitted and not isinstance(exc, _DrainedWriteFailure))): + quarantined = True + self._retain_uncertain({"tensors": tensors, "keys": keys, + "registered": tuple(registered), "bytes": sum(sizes)}) + raise CaptureWriterUncertain(str(exc)) from None self._remove_many_quiet(keys) raise finally: - for ptr in registered: - try: - store.unregister_buffer(ptr) - except Exception: - pass - - if statuses is None: - statuses = [0] * len(keys) - if len(statuses) != len(keys): - self._remove_many_quiet(keys) - raise RuntimeError( - f"spec-capture batch_put_from returned {len(statuses)} statuses " - f"for {len(keys)} keys" - ) - failed = [ - (key, status) - for key, status in zip(keys, statuses) - if status is not None and int(status) < 0 - ] - if failed: - self._remove_many_quiet(keys) - raise RuntimeError( - f"spec-capture batch_put_from failed for {len(failed)}/{len(keys)} " - f"keys; first={failed[0]}" - ) + unregister_started = time.monotonic_ns() if diagnostic is not None else 0 + if not quarantined: + for ptr in registered: + try: + store.unregister_buffer(ptr) + except Exception: + pass + if diagnostic is not None: + diagnostic["p0_v1_sink_unregister_ms"] = (time.monotonic_ns() - unregister_started) / 1e6 + diagnostic["p0_v1_sink_quarantined"] = quarantined if timing_enabled: logger.info( diff --git a/specforge/config/schema.py b/specforge/config/schema.py index fdf2332919..4a3bc8d026 100644 --- a/specforge/config/schema.py +++ b/specforge/config/schema.py @@ -141,7 +141,7 @@ #: ``sglang_*`` fields that carry raw argv tokens instead of one flag. SGLANG_PASSTHROUGH_FIELDS = frozenset({"sglang_extra_args"}) #: Capture-server environment the managed-local launcher sets itself. -_OWNED_SERVER_ENV_PREFIXES = ("MOONCAKE_", "DISAGG_") +_OWNED_SERVER_ENV_PREFIXES = ("MOONCAKE_", "DISAGG_", "SPECFORGE_HS_") _OWNED_SERVER_ENV = { "CUDA_VISIBLE_DEVICES": "capture_servers[].cuda_visible_devices", "ASCEND_RT_VISIBLE_DEVICES": "capture_servers[].cuda_visible_devices", @@ -419,6 +419,7 @@ class DataConfig(StrictConfigModel): #: offline evaluation — directory of precomputed hidden-state .ckpt files. eval_hidden_states_path: str = "" max_length: int = Field(default=2048, gt=0) + overlength_policy: Literal["truncate", "drop"] = "truncate" chat_template: str = "llama3" is_preformatted: bool = False train_only_last_turn: bool = False @@ -432,6 +433,8 @@ class DataConfig(StrictConfigModel): @model_validator(mode="after") def _exactly_one_source(self): + if self.overlength_policy == "drop" and not self.prompts_path: + raise ValueError("data.overlength_policy=drop requires native data.prompts_path") sources = [ bool(self.train_data_path), bool(self.prompts_path), @@ -495,6 +498,9 @@ class RuntimeConfig(StrictConfigModel): """Streaming bounds shared by unified disaggregated producer roles.""" producer_lease: int = Field(default=8, gt=0) + producer_backpressure_poll_s: float = Field(default=0.2, gt=0, allow_inf_nan=False) + consumer_ref_poll_s: float = Field(default=0.1, gt=0, allow_inf_nan=False) + ref_distributor_poll_s: float = Field(default=0.05, gt=0, allow_inf_nan=False) consumer_dispatch: Literal[ "round_robin", "cost_balanced", @@ -509,6 +515,14 @@ class RuntimeConfig(StrictConfigModel): feature_store_max_resident_bytes: Optional[int] = Field(default=None, gt=0) feature_store_max_quarantined_bytes: int = Field(default=8 << 30, ge=0) + @field_validator("producer_backpressure_poll_s", "consumer_ref_poll_s", + "ref_distributor_poll_s", mode="before") + @classmethod + def _poll_seconds_numeric(cls, value): + if isinstance(value, bool): + raise ValueError("polling intervals must be positive finite numbers") + return value + @model_validator(mode="after") def _validate_watermarks(self): resolve_ref_watermarks( @@ -841,6 +855,17 @@ class DisaggregatedDeploymentConfig(StrictConfigModel): #: backend: managed_local probes each capture server's interpreter while #: planning; the producer reads an external server's ``/server_info``. server_capture: Literal["auto", "plugin"] = "auto" + #: Native HS protocol and content-cache reuse are separate opt-ins so + #: cache-off/on comparisons share the same publication/ownership path. + hs_capture: bool = False + hs_prefix_cache: bool = False + hs_chunk_tokens: int = Field(default=256, gt=0) + hs_cache_bytes: int = Field(default=1 << 30, gt=0) + hs_pinned_bytes: int = Field(default=256 << 20, gt=0) + hs_output_bytes: int = Field(default=512 << 20, gt=0) + hs_device_bytes: int = Field(default=256 << 20, gt=0) + hs_max_workers: int = Field(default=4, gt=0) + hs_attempt_capacity: int = Field(default=4096, gt=0) mooncake_metadata_server: Optional[str] = None mooncake_master_server_addr: Optional[str] = None mooncake_local_hostname: Optional[str] = None @@ -880,6 +905,10 @@ class DisaggregatedDeploymentConfig(StrictConfigModel): @model_validator(mode="after") def _validate_store(self): + if self.hs_prefix_cache and not self.hs_capture: + raise ValueError("hs_prefix_cache requires hs_capture") + if self.hs_capture and self.backend != "mooncake": + raise ValueError("HS capture requires backend=mooncake") if not self.control_dir: raise ValueError("deployment.disaggregated.control_dir must not be empty") if self.managed_local is not None: @@ -982,7 +1011,9 @@ class TrainingConfig(StrictConfigModel): #: Joint logits loss; independent of objective checkpointing. fused_logits_loss: bool = True learning_rate: float = Field(default=1e-4, gt=0.0) - lr_scheduler: Literal["cosine", "constant"] = "cosine" + lr_scheduler: Literal["cosine", "constant", "torchspec_cosine"] = "cosine" + weight_decay: float = Field(default=0.0, ge=0.0) + min_lr: float = Field(default=0.0, ge=0.0) warmup_ratio: float = Field(default=0.015, ge=0.0, le=1.0) max_grad_norm: float = Field(default=0.5, gt=0.0) #: Keep FP32 Adam masters and moments on CPU while the trainable draft @@ -1074,6 +1105,10 @@ class TrainingConfig(StrictConfigModel): @model_validator(mode="after") def _validate_training_shape(self): + if self.min_lr > self.learning_rate: + raise ValueError("training.min_lr must not exceed training.learning_rate") + if self.min_lr and self.lr_scheduler == "constant": + raise ValueError("training.min_lr requires a cosine scheduler") if not 0.0 <= self.dpace_alpha <= 1.0: raise ValueError("training.dpace_alpha must be in [0, 1]") if not 0.0 < self.down_sample_ratio <= 1.0: diff --git a/specforge/data/prompt_builder.py b/specforge/data/prompt_builder.py index 30a7c5c9f2..3fd84161b2 100644 --- a/specforge/data/prompt_builder.py +++ b/specforge/data/prompt_builder.py @@ -32,6 +32,8 @@ def prepare_prompt_tasks( min_loss_tokens: int = 1, max_prompts: int | None = None, loss_mask_filter: Callable[[Sequence[int]], bool] | None = None, + overlength_policy: str = "truncate", + filter_report: dict[str, Any] | None = None, ) -> Sequence[PromptTaskDict]: """Prepare runtime prompt dictionaries from a JSONL file. @@ -43,11 +45,17 @@ def prepare_prompt_tasks( lazy random-access sequence so large Arrow datasets are not expanded into Python token lists before rollout starts. - ``max_prompts`` caps input records before preprocessing and filtering on - both paths. Rejected records count toward the cap and are not replaced, - so fewer prompts may be returned. ``None`` and ``0`` mean no cap. + Native ``overlength_policy="drop"`` rejects full P+G records longer than + ``max_length`` before any slicing. Equal-length records remain unchanged. + In this mode, ``max_prompts`` caps accepted prompts for compatibility with + existing filtered datasets. Raw conversations and native ``truncate`` cap + input records before preprocessing and filtering; rejected rows consume + that quota. ``None`` and ``0`` mean no cap in either mode. + ``filter_report`` optionally receives the filter counts and dropped IDs. """ + if overlength_policy not in {"truncate", "drop"}: + raise ValueError("overlength_policy must be truncate or drop") _validate_options( max_length=max_length, min_loss_tokens=min_loss_tokens, @@ -77,15 +85,22 @@ def prepare_prompt_tasks( (record, f"{path_string}:{line_number}") for line_number, record in _iter_records(path_string) ) - if limit is not None: + if limit is not None and overlength_policy == "truncate": rows = islice(rows, limit) return _materialize_prompt_tasks( rows, max_length=max_length, min_loss_tokens=min_loss_tokens, loss_mask_filter=loss_mask_filter, + accepted_limit=limit if overlength_policy == "drop" else None, + overlength_policy=overlength_policy, + filter_report=filter_report, ) + if overlength_policy == "drop": + raise ValueError( + "overlength_policy=drop requires pre-tokenized input_ids/loss_mask" + ) return _prepare_raw_prompts( path_string, tokenizer, @@ -195,22 +210,51 @@ def _materialize_prompt_tasks( max_length: int, min_loss_tokens: int, loss_mask_filter: Callable[[Sequence[int]], bool] | None, + accepted_limit: int | None = None, + overlength_policy: str = "truncate", + filter_report: dict[str, Any] | None = None, ) -> list[PromptTaskDict]: + report = filter_report if filter_report is not None else {} + report.update( + policy=overlength_policy, + max_length=max_length, + rows_seen=0, + accepted=0, + dropped_overlength=0, + dropped_other=0, + kept_input_tokens=0, + kept_loss_tokens=0, + dropped_samples=[], + ) prompts: list[PromptTaskDict] = [] for record, source in rows: + report["rows_seen"] += 1 + before_overlength = report["dropped_overlength"] prompt = _prompt_from_record( record, source=source, max_length=max_length, min_loss_tokens=min_loss_tokens, + overlength_policy=overlength_policy, + filter_report=report, ) if prompt is None: + if report["dropped_overlength"] == before_overlength: + report["dropped_other"] += 1 continue if loss_mask_filter is not None and not loss_mask_filter( prompt["payload"]["loss_mask"] ): + report["dropped_other"] += 1 continue prompts.append(prompt) + report["accepted"] += 1 + report["kept_input_tokens"] += len(prompt["payload"]["input_ids"]) + report["kept_loss_tokens"] += sum(prompt["payload"]["loss_mask"]) + if accepted_limit is not None and len(prompts) >= accepted_limit: + break + if overlength_policy == "drop": + print("[prompt-filter] " + json.dumps(report, ensure_ascii=False), flush=True) return prompts @@ -220,6 +264,8 @@ def _prompt_from_record( source: str, max_length: int, min_loss_tokens: int, + overlength_policy: str = "truncate", + filter_report: dict[str, Any] | None = None, ) -> PromptTaskDict | None: if "input_ids" not in record or "loss_mask" not in record: raise ValueError(f"{source} must contain both input_ids and loss_mask") @@ -238,6 +284,19 @@ def _prompt_from_record( if not input_ids: raise ValueError(f"{source} contains an empty token sequence") + if overlength_policy == "drop" and len(input_ids) > max_length: + if filter_report is not None: + filter_report["dropped_overlength"] += 1 + filter_report["dropped_samples"].append( + { + "id": record.get("id"), + "source": source, + "length": len(input_ids), + "loss_tokens": sum(loss_mask), + } + ) + return None + input_ids = input_ids[:max_length] loss_mask = loss_mask[:max_length] if sum(loss_mask) < min_loss_tokens: diff --git a/specforge/inference/adapters/server_capture.py b/specforge/inference/adapters/server_capture.py index 7126a3f294..3d3e4e55f8 100644 --- a/specforge/inference/adapters/server_capture.py +++ b/specforge/inference/adapters/server_capture.py @@ -27,6 +27,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import json import logging import threading @@ -200,6 +202,9 @@ def __init__( target_model_version: str = "unknown", server_capture: str = "plugin", server_info_fn: Optional[Callable[[str, float], Mapping[str, Any]]] = None, + hs_capture: bool = False, + hs_prefix_cache: bool = False, + control_fn: Optional[Callable[..., Any]] = None, ) -> None: if server_capture not in ("auto", *SERVER_CAPTURE_BACKENDS): raise ValueError( @@ -238,6 +243,16 @@ def __init__( self.request_input_adapter = request_input_adapter self.timeout_s = timeout_s self.post_fn = post_fn or _SessionPost() + if hs_prefix_cache and not hs_capture: + raise ValueError("hs_prefix_cache requires explicit hs_capture opt-in") + self.hs_capture = hs_capture + self.hs_prefix_cache = hs_prefix_cache + from specforge.inference.capture_protocol import CaptureProtocol + + self._capture_protocol = ( + CaptureProtocol(base_url, control_fn or self.post_fn, timeout_s) + if hs_capture else None + ) self.target_model_version = target_model_version #: ``auto`` until the first request reads the server's backend. self.server_capture = server_capture @@ -382,7 +397,7 @@ def _ref_from_result( run_id=self.run_id, source_task_id=task.task_id, feature_store_uri=f"mooncake://{result['store_id']}/{sample_id}", - feature_keys={n: f"{sample_id}/{n}" for n in specs}, + feature_keys=(dict(result["physical_keys"]) if result.get("publication_ownership") == "attempt-v1" else {n: f"{sample_id}/{n}" for n in specs}), feature_specs=specs, strategy=self.strategy, schema_version=SCHEMA_VERSION, @@ -408,6 +423,15 @@ def _ref_from_result( ), }, ) + if result.get("protocol_version") == 1: + ref.metadata["capture_fence"] = { + **{name: result[name] for name in ("server_epoch", "store_id", "sample_id", "gen", "attempt_id", "reservation_token")}, + "protocol_version": 1, "control_url": self.base_url + "/spec_capture/control", + "timeout_s": self.timeout_s, + **{name: result[name] for name in ('object_key_layout', 'publication_ownership', 'physical_keys', 'writer_drained', 'state') if name in result}, + } + ref.metadata["capture_diagnostics"] = {name: result.get(name) for name in + ("H", "L", "N", "span", "execution_epoch", "rank", "reused_tokens", "capture_namespace", "capture_signature", "timings")} if "arena" in result: # Arena objects stay on the capture server; currently only online # NVLink capture uses the arena. @@ -433,16 +457,184 @@ def produce_refs( transport-level error raises — the worker fails the whole lease batch retryable, mirroring ``generate_features`` semantics. """ + prepared = self.prepare_refs_batches([tasks], capture=capture) + self.register_prepared_batches(prepared) + return self.produce_prepared(prepared[0], capture=capture) + + def prepare_refs_batches(self, task_batches, *, capture: CaptureConfig): + """Build bounded lease groups without registering a writer yet.""" + self._resolve_server_capture() + capability = self._capture_protocol.capability() if self._capture_protocol is not None else None + if self.hs_capture: + if capability is None or capability.get("published_ownership") != "attempt-v1": + raise RuntimeError("HS capture requires native plugin attempt-v1 capability") + if capability.get("cache_enabled") is not self.hs_prefix_cache: + raise RuntimeError("HS cache setting disagrees with server capability") + if capability is not None: + required = ("prepare_capture_retry", "track_capture_fence") + if any(not callable(getattr(self.store, name, None)) for name in required): + raise TypeError("negotiated capture requires attempt-fenced store") + requested_layers = tuple(capture.aux_hidden_state_layer_ids or ()) + schema = {item["name"]: item for item in capability["schema"]} + if requested_layers and schema.get("aux", {}).get("width") != capture.expected_aux_width: + raise CaptureMismatchError("server capability aux width mismatch") + if self.schema.last_hidden_feature is not None and schema.get("last_hidden", {}).get("width") != capture.target_hidden_size: + raise CaptureMismatchError("server capability last hidden width mismatch") + if requested_layers and tuple(schema.get("aux", {}).get("layer_ids", ())) != requested_layers: + raise CaptureMismatchError("server capability aux layer order mismatch") + prepared = [] + for tasks in task_batches: + body = self._request_inputs(tasks) + body["extra_key"] = [uuid.uuid4().hex for _ in tasks] + payloads = [self._spec_capture_payload(task) for task in tasks] + if capability is not None: + for payload in payloads: + self.store.prepare_capture_retry(payload["sample_id"]) + if capability["cache_enabled"]: + body["extra_key"] = [capability["capture_namespace"]] * len(tasks) + prepared.append(dict(tasks=tasks, body=body, payloads=payloads, + capability=capability, + estimated_bytes=self._output_bytes(tasks, capability))) + return prepared + + def _output_bytes(self, tasks, capability): + # The negotiated schema fixes the complete engine tensor width/dtype. + # Passthrough tensors are int64 and have client-declared trailing dims. + if capability is None: + return None + schema = {item["name"]: item for item in capability["schema"]} + per_token = 0 + for artifact, feature in (("aux", self.schema.aux_feature), + ("last_hidden", self.schema.last_hidden_feature)): + if feature is not None: + item = schema[artifact] + width = int(item["width"]) + if width < 1 or item["dtype"] not in _DTYPE_BYTES: + raise ValueError("invalid negotiated capture output schema") + per_token += width * _DTYPE_BYTES[item["dtype"]] + for _name, _key, trailing in self.schema.passthrough: + per_token += _spec_nbytes((1, *trailing), "int64") + if self.schema.attention_mask_feature is not None: + per_token += 8 + return sum(len(task.payload["input_ids"]) * per_token for task in tasks) + + def registration_capacity(self): + """Return the negotiated endpoint's atomic begin bound before leasing.""" + if self._capture_protocol is None: + return None + return self._registration_capacity(self._capture_protocol.capability()) + + @staticmethod + def _registration_capacity(capability): + if capability is None: + return None + capacity = capability.get("limits", {}).get("attempt_capacity") + if capacity is not None and (isinstance(capacity, bool) or + not isinstance(capacity, int) or capacity < 1): + raise RuntimeError("invalid capture attempt capacity") + return capacity + + def register_prepared_batches(self, prepared): + """Coalesce all currently ready leases into one endpoint begin_batch.""" + if not prepared: + return + capability = prepared[0]["capability"] + if any(item["capability"] != capability for item in prepared): + raise RuntimeError("capture capability changed within registration batch") + if capability is None: + return + payloads = [payload for item in prepared for payload in item["payloads"]] + tasks = [task for item in prepared for task in item["tasks"]] + capacity = self._registration_capacity(capability) + if capacity is not None and len(tasks) > capacity: + # No attempt identity or local begin intent exists yet. The producer + # splits ready groups before this boundary; direct callers receive a + # local error rather than an ambiguous partial remote reservation. + raise ValueError("registration batch exceeds capture attempt capacity") + requests = [] + for item in prepared: + item["begin_control_url"] = self._capture_protocol.url + retryable_payloads = { + id(payload) for item in prepared if item.get("registration_refused") + for payload in item["payloads"] + } + for task, payload in zip(tasks, payloads): + names = [*payload["features"].values(), *(x["name"] for x in payload.get("passthrough", ()))] + # Freeze the begin identity before crossing HTTP. Even a lost reply + # must not let a late cleanup retarget a peer's replacement writer. + if "attempt_id" in payload and id(payload) not in retryable_payloads: + raise RuntimeError("prepared capture registration cannot be replayed") + payload.update(attempt_id=payload.get("attempt_id", uuid.uuid4().hex), + server_epoch=capability["server_epoch"], replace=False) + from specforge.inference.capture_protocol import request_digest + payload["request_digest"] = request_digest(task.payload["input_ids"], payload) + requests.append((payload, task.payload["input_ids"], payload["attempt_id"], names)) + from specforge.inference.capture_protocol import CaptureCapacityDeclined + try: + fences = self.store.begin_capture_attempts(self._capture_protocol, capability, requests) + except CaptureCapacityDeclined: + for item in prepared: + item["registration_refused"] = True + raise + for item in prepared: + item["registration_refused"] = False + for payload, fence in zip(payloads, fences): + payload.update({name: capability[name] for name in ("server_epoch", "capture_namespace", "capture_signature")}) + payload.update({name: fence[name] for name in ("protocol_version", "attempt_id", "reservation_token")}) + if fence.get("object_key_layout") == "attempt-v1": + payload["object_key_layout"] = "attempt-v1" + + @staticmethod + def _verify_hs_feature_metadata(result, payload, capability): + """Check the complete negotiated schema, including passthrough dtype.""" + schema = {item["name"]: item for item in capability["schema"]} + n = result.get("N") + if isinstance(n, bool) or not isinstance(n, int) or n < 1: + raise RuntimeError("negotiated capture returned invalid token count") + expected = { + name: ((1, n, int(schema[artifact]["width"])), schema[artifact]["dtype"]) + for artifact, name in payload["features"].items() + } + expected.update({item["name"]: (tuple(item["shape"]), item["dtype"]) + for item in payload.get("passthrough", ())}) + actual = result.get("features", {}) + if set(actual) != set(expected): + raise RuntimeError("negotiated capture feature names mismatch") + for name, (shape, dtype) in expected.items(): + if tuple(actual[name].get("shape", ())) != shape or actual[name].get("dtype") != dtype: + raise RuntimeError("negotiated capture feature shape/dtype mismatch: " + name) + + def unreturned_prepared(self, prepared, refs): + """Retain only per-task failures for exact-attempt retirement.""" + returned = {ref.sample_id for ref in refs} + pairs = [(task, payload) for task, payload in zip(prepared["tasks"], prepared["payloads"]) + if payload["sample_id"] not in returned] + if not pairs: + return None + tasks, payloads = map(list, zip(*pairs)) + return {**prepared, "tasks": tasks, "payloads": payloads, + "estimated_bytes": self._output_bytes(tasks, prepared["capability"])} + + def discard_prepared_batches(self, prepared): + # Resolve uncertain begin replies using the store's immutable original + # intent, then retire that exact writer before releasing byte credits. + for item in prepared: + for payload in item["payloads"]: + if "attempt_id" not in payload: + continue # This lease never registered a server writer. + self.store.discard_capture_attempt( + payload["sample_id"], server_epoch=payload["server_epoch"], + attempt_id=payload["attempt_id"], control_url=item["begin_control_url"], + reservation_token=payload.get("reservation_token"), + ) + + def produce_prepared(self, prepared, *, capture: CaptureConfig): + tasks = prepared["tasks"] + body = prepared["body"] + capture_payloads = prepared["payloads"] + capability = prepared["capability"] server_capture = self._resolve_server_capture() - body = self._request_inputs(tasks) - # A fresh cache namespace forces a full prefill on every attempt while - # leaving SGLang free to use the radix data structure required by hybrid - # targets. Retries need a new key because the prior attempt may have - # populated its namespace before the response was lost. - body["extra_key"] = [uuid.uuid4().hex for _ in tasks] sampling_params = {"temperature": 0.0, "max_new_tokens": 0} - capture_payloads = [self._spec_capture_payload(t) for t in tasks] - # custom_params values are JSON scalars: one encoded spec per request. body["sampling_params"] = [ { **sampling_params, @@ -463,9 +655,14 @@ def produce_refs( feature_names=feature_names, server_url=self.base_url, ) - rows = self.post_fn( - f"{self.base_url}/generate", json_body=body, timeout=self.timeout_s - ) + try: + rows = self.post_fn( + f"{self.base_url}/generate", json_body=body, timeout=self.timeout_s + ) + except Exception as exc: + if self._capture_protocol is not None and getattr(getattr(exc, 'response', None), 'status_code', None) == 409: + self._capture_protocol.capability(refresh=True) + raise rows = _flatten_list_wrappers(rows) if len(rows) != len(tasks): raise RuntimeError( @@ -475,7 +672,7 @@ def produce_refs( capture_hint = _CAPTURE_HINTS[server_capture] out: List[Union[Any, ServerCaptureFailure]] = [] successful_refs = [] - for task, row in zip(tasks, rows): + for task, row, payload in zip(tasks, rows, capture_payloads): if not isinstance(row, dict): raise RuntimeError( "spec-capture server returned a non-object row for task " @@ -526,7 +723,31 @@ def produce_refs( f"task {task.task_id}: {actual_identity} != " f"{expected_identity}" ) + if capability is not None: + for name in ("protocol_version", "server_epoch", "capture_namespace", "capture_signature", "attempt_id", "reservation_token"): + if result.get(name) != payload.get(name): + if name == 'server_epoch': + self._capture_protocol.invalidate_epoch(result.get(name)) + raise RuntimeError("negotiated capture response identity mismatch: " + name) + if self.hs_capture: + if result.get("request_digest") != payload.get("request_digest"): + raise RuntimeError("negotiated capture response request digest mismatch") + self._verify_hs_feature_metadata(result, payload, capability) + if capability.get('published_ownership') == 'attempt-v1': + if result.get('publication_ownership') != 'attempt-v1' or result.get('state') != 'PUBLISHED' or not result.get('writer_drained'): + raise RuntimeError('capture response omitted drained ownership transfer') + from specforge.inference.capture_protocol import attempt_keys + names = [*payload['features'].values(), *(x['name'] for x in payload.get('passthrough', ()))] + keys = attempt_keys(result, names) + if result.get('physical_keys') != keys or any(result.get('features', {}).get(name, {}).get('key') != key for name, key in keys.items()): + raise RuntimeError('capture response immutable keys mismatch') + if self.hs_capture: + N, L = len(task.payload["input_ids"]), result.get("L") + if isinstance(L, bool) or not isinstance(L, int) or not 0 <= L < N or result.get("N") != N or result.get("span") != [L, N]: + raise RuntimeError("negotiated capture returned invalid absolute span") ref = self._ref_from_result(task, result, capture) + if capability is not None and ref.estimated_bytes > self._output_bytes([task], capability): + raise RuntimeError("capture output exceeded negotiated byte reservation") # A capture shorter than the prompt is corrupt: it means cache # isolation or another full-prefill invariant failed despite the # request's fresh namespace. @@ -538,7 +759,7 @@ def produce_refs( } if short: self.store.adopt(ref) - self.store.abort(ref.sample_id, reason="seq-len-mismatch") + abort_feature_ref(self.store, ref, reason="seq-len-mismatch") out.append( ServerCaptureFailure( task_id=task.task_id, @@ -578,7 +799,7 @@ def produce_refs( # Loud boundary failure; free the server-written keys so a # mismatched sample is never consumable. self.store.adopt(ref) - self.store.abort(ref.sample_id, reason=f"contract:{exc}") + abort_feature_ref(self.store, ref, reason=f"contract:{exc}") out.append( ServerCaptureFailure( task_id=task.task_id, reason=str(exc), retryable=False diff --git a/specforge/inference/capture_protocol.py b/specforge/inference/capture_protocol.py new file mode 100644 index 0000000000..05d0d59d06 --- /dev/null +++ b/specforge/inference/capture_protocol.py @@ -0,0 +1,240 @@ +"""Version-one capture control client and serializable sample writer fences.""" +from __future__ import annotations + +import hashlib +import json +import threading +import time +import uuid +from dataclasses import dataclass, field + + +class CaptureControlRejected(RuntimeError): + """The server explicitly rejected the operation without granting a permit.""" + + +class CaptureCapacityDeclined(CaptureControlRejected): + """An atomic begin was refused before granting any reservation.""" + + +def request_digest(tokens, spec): + value = {"tokens": list(tokens), "features": spec.get("features", {}), + "passthrough": spec.get("passthrough", [])} + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def http_control(url, json_body, timeout): + import requests + response = requests.post(url, json=json_body, timeout=timeout) + response.raise_for_status() + return response.json() + + +class CaptureProtocol: + def __init__(self, base_url, post, timeout): + self.url = base_url.rstrip('/') + '/spec_capture/control' + self.post = post + self.timeout = timeout + self.negotiated = False + self._capability = None + self._capability_known = False + self._capability_lock = threading.RLock() + + def call(self, body): + try: + result = self.post(self.url, json_body={"protocol_version": 1, **body}, timeout=self.timeout) + except Exception as exc: + if body.get('action') != 'capability' and getattr(getattr(exc, 'response', None), 'status_code', None) == 409: + # Refresh discovery only. Never replay an uncertain mutation. + try: + self.capability(refresh=True) + except Exception: + pass # Keep the original rejection recognizable to its owner. + raise + if (isinstance(result, dict) and body.get('action') in ('begin', 'begin_batch') + and result.get('error_code') == 'capture_capacity_declined' + and result.get('reservation_granted') is False): + raise CaptureCapacityDeclined(str(result.get('error', 'capture capacity exhausted'))) + if not isinstance(result, dict) or result.get('error'): + raise RuntimeError('capture control failed: ' + repr(result)) + return result + + def capability(self, *, refresh=False): + with self._capability_lock: + if self._capability_known and not refresh: + return dict(self._capability) if self._capability is not None else None + return self._discover_capability() + + def invalidate_epoch(self, server_epoch): + if self._capability is not None and self._capability['server_epoch'] != server_epoch: + self.capability(refresh=True) + + def _discover_capability(self): + try: + result = self.call({'action': 'capability'}) + except Exception as exc: + status = getattr(getattr(exc, 'response', None), 'status_code', None) + if status == 404 and not self.negotiated: + self._capability = None + self._capability_known = True + return None + raise + if result.get('protocol_version') != 1: + if self.negotiated: + raise RuntimeError('negotiated capture protocol disappeared') + self._capability = None + self._capability_known = True + return None + required = ('server_epoch', 'capture_namespace', 'capture_signature', 'schema', 'cache_enabled') + if any(name not in result for name in required): + raise RuntimeError('incomplete capture capability') + self.negotiated = True + self._capability_known = True + self._capability = dict(result) + return dict(result) + + def begin(self, capability, spec, tokens, attempt_id): + reply = self.call({'action': 'begin', **self._begin_body(capability, spec, tokens, attempt_id)}) + return self._begin_wire(capability, spec, attempt_id, reply) + + def _begin_body(self, capability, spec, tokens, attempt_id): + return {**{name: spec[name] for name in ('store_id', 'sample_id', 'gen')}, + 'server_epoch': capability['server_epoch'], 'attempt_id': attempt_id, + 'request_digest': request_digest(tokens, spec), + 'publication_ownership': capability.get('published_ownership')} + + def begin_batch(self, capability, requests): + if not capability.get('begin_batch'): + return [self.begin(capability, spec, tokens, attempt_id) for spec, tokens, attempt_id in requests] + tasks = [self._begin_body(capability, *request) for request in requests] + result = self.call({'action': 'begin_batch', 'server_epoch': capability['server_epoch'], 'tasks': tasks}) + replies = result.get('attempts') + if not isinstance(replies, list) or len(replies) != len(requests): + raise RuntimeError('capture begin_batch count mismatch') + return [self._begin_wire(capability, spec, attempt_id, reply) + for (spec, _, attempt_id), reply in zip(requests, replies)] + + def _begin_wire(self, capability, spec, attempt_id, reply): + identity = {name: spec[name] for name in ('store_id', 'sample_id', 'gen')} + for name, value in {**identity, 'attempt_id': attempt_id, 'server_epoch': capability['server_epoch']}.items(): + if reply.get(name) != value: + raise RuntimeError('capture begin identity mismatch: ' + name) + if not reply.get('reservation_token'): + raise RuntimeError('capture begin omitted server reservation') + return {**identity, 'protocol_version': 1, 'server_epoch': capability['server_epoch'], + 'attempt_id': attempt_id, 'reservation_token': reply['reservation_token'], + 'control_url': self.url, 'timeout_s': self.timeout, + **({'object_key_layout': 'attempt-v1'} if reply.get('object_key_layout') == 'attempt-v1' else {})} + + +@dataclass +class WriterFence: + """A sample's writer identity; serializable fields travel on every SampleRef.""" + wire: dict + names: tuple + post: object = http_control + writer_drained: bool = False + transferred: bool = False + removed: bool = False + acknowledged: bool = False + adopted: bool = False + cleanup_owner: str = field(default_factory=lambda: uuid.uuid4().hex) + cleanup_token: str | None = None + remote_retired: bool = False + lock: object = field(default_factory=threading.RLock) + + @property + def token(self): + return self.wire['reservation_token'] + + def control(self, action, **values): + body = {name: self.wire[name] for name in ('server_epoch', 'store_id', 'sample_id', 'gen', 'attempt_id', 'reservation_token')} + try: + result = self.post(self.wire['control_url'], json_body={'protocol_version': 1, 'action': action, **body, **values}, + timeout=min(float(self.wire.get('timeout_s', 30.0)), 30.0)) + except Exception as exc: + if getattr(exc, 'capture_control_rejected', False) or getattr(getattr(exc, 'response', None), 'status_code', None) == 409: + raise CaptureControlRejected(str(exc)) from None + raise + if isinstance(result, dict) and result.get('error'): + raise CaptureControlRejected('capture fence control rejected: ' + str(result['error'])) + if not isinstance(result, dict): + raise RuntimeError('capture fence control returned invalid response') + if any(result.get(name) != value for name, value in body.items()): + raise RuntimeError('capture fence identity changed') + return result + + def accept_transfer(self, certificate): + if certificate.get('publication_ownership') != 'attempt-v1': + return False + if self.wire.get('object_key_layout') != 'attempt-v1' or certificate.get('state') != 'PUBLISHED' or certificate.get('writer_drained') is not True: + raise RuntimeError('capture ownership transfer before writer drain') + if any(certificate.get(name) != self.wire[name] for name in + ('server_epoch', 'store_id', 'sample_id', 'gen', 'attempt_id', 'reservation_token')): + raise RuntimeError('capture ownership certificate identity mismatch') + expected = attempt_keys(self.wire, self.names) + if certificate.get('physical_keys') != expected: + raise RuntimeError('capture ownership certificate keys mismatch') + self.wire.update(publication_ownership='attempt-v1', physical_keys=expected, + writer_drained=True, state='PUBLISHED') + self.transferred = self.writer_drained = True + return True + + def drain(self, *, cancel=True): + if self.writer_drained: + return + deadline = time.monotonic() + float(self.wire.get('timeout_s', 30.0)) + while True: + result = self.control('resolve', cancel=cancel) + if result.get('writer_drained'): + self.accept_transfer(result.get("publication_certificate", result)) + self.writer_drained = True + return + if time.monotonic() >= deadline: + raise TimeoutError('capture writer has not drained; sample retained') + time.sleep(0.01) + + def acquire_read(self, reader_owner): + reply = self.control('read_acquire', reader_owner=reader_owner) + if not reply.get('reader_token') or reply.get('reader_owner') != reader_owner: + raise RuntimeError('server did not grant exact read ownership') + return reply['reader_token'] + + def release_read(self, reader_owner, reader_token): + reply = self.control('read_release', reader_owner=reader_owner, reader_token=reader_token) + if not reply.get('reader_released') or reply.get('reader_token') != reader_token: + raise RuntimeError('server did not release exact read ownership') + + def acquire_cleanup(self): + # Only a complete immutable-key ownership certificate bypasses permits. + self.drain(cancel=True) + if self.transferred: + return + reply = self.control('cleanup_acquire', cleanup_owner=self.cleanup_owner) + if reply.get('state') == 'RETIRED': + self.remote_retired = self.removed = self.acknowledged = True + return + if not reply.get('cleanup_token') or reply.get('cleanup_owner') != self.cleanup_owner: + raise RuntimeError('server did not grant exact cleanup ownership') + self.cleanup_token = reply['cleanup_token'] + + def acknowledge(self): + if self.acknowledged: + return + if not self.writer_drained or not self.removed: + raise RuntimeError('cleanup acknowledgement before writer/remove drain') + if self.transferred: + self.acknowledged = True + return + if not self.cleanup_token: + raise RuntimeError('cleanup acknowledgement without server permit') + result = self.control('cleanup_done', cleanup_token=self.cleanup_token) + if result.get('state') != 'RETIRED': + raise RuntimeError('capture cleanup was not retired') + self.acknowledged = True + + +def attempt_keys(wire, names): + """Immutable physical identity; a stale process can only address its attempt.""" + base = f"{wire['store_id']}/{wire['sample_id']}/g{int(wire['gen'])}/attempt/{wire['server_epoch']}/{wire['reservation_token']}" + return {str(name): base + '/' + str(name) for name in names} diff --git a/specforge/inference/rollout_worker.py b/specforge/inference/rollout_worker.py index eebf2a8035..d13e5ac8fa 100644 --- a/specforge/inference/rollout_worker.py +++ b/specforge/inference/rollout_worker.py @@ -123,14 +123,34 @@ def _record_commit(self, count: int) -> None: def _sample_id(self, task: PromptTask) -> str: return f"{self.run_id}:{task.task_id}" - def run_once(self, max_tasks: int) -> List[SampleRef]: + def lease_tasks(self, max_tasks: int) -> List[PromptTask]: + """Lease once, including queued preparation in worker health.""" with self._health_lock: if self._state in ("stopped", "draining"): return [] tasks = self.controller.lease_prompt_tasks(self.worker_id, max_tasks) + if tasks: + self._begin_batch(len(tasks)) + return tasks + + def fail_leased(self, tasks, reason: str, *, unhealthy: bool = True) -> None: + if unhealthy: + self._mark_unhealthy(reason) + self.controller.fail_prompt_tasks( + self.worker_id, [task.task_id for task in tasks], + reason=reason, retryable=True, + ) + self._finish_batch(len(tasks)) + + def run_once(self, max_tasks: int) -> List[SampleRef]: + return self.run_leased(self.lease_tasks(max_tasks), max_tasks=max_tasks) + + def run_leased(self, tasks: List[PromptTask], *, max_tasks: Optional[int] = None, + prepared=None) -> List[SampleRef]: + """Execute an existing lease; preparation never leases a second time.""" if not tasks: return [] - self._begin_batch(len(tasks)) + max_tasks = len(tasks) if max_tasks is None else max_tasks # Legacy colocated target-TP training used a drop-last target batch. # A partial target batch would give ranks different local batch sizes # and break the all-rank draft/FSDP step contract. @@ -151,6 +171,10 @@ def run_once(self, max_tasks: int) -> List[SampleRef]: "target-TP batch partitioning is supported for colocated " "feature capture only" ) + if prepared is not None: + produce_refs = lambda _tasks, *, capture: self.feature_source.produce_prepared( + prepared, capture=capture + ) return self._run_ref_source(tasks, produce_refs) try: diff --git a/specforge/launch.py b/specforge/launch.py index 3a0b19c881..a892a67402 100644 --- a/specforge/launch.py +++ b/specforge/launch.py @@ -10,6 +10,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import logging import os from typing import Any, Callable, List, Mapping, Optional, Tuple @@ -386,8 +388,8 @@ def _publish_refs_with_cleanup( cleanup_errors = [] for ref in transaction.unpublished_refs: try: - feature_store.abort( - ref.sample_id, + abort_feature_ref( + feature_store, ref, reason="producer-ref-publication-failed", ) except Exception as cleanup_exc: @@ -402,6 +404,11 @@ def _publish_refs_with_cleanup( f"{cleanup_errors}" ) from publish_exc raise + finally: + relinquish = getattr(feature_store, "relinquish_published", None) + if callable(relinquish): + for ref in transaction.published_refs: + relinquish(ref) def _epoch_online_prompts( @@ -838,8 +845,12 @@ def build_disagg_online_producer( server-capture transport; a *sequence* of sources fans out to one logical worker per source (the multi-server topology). Each logical worker keeps ``producer_concurrency`` capture calls in flight and replenishes a slot as - soon as that call completes. There is intentionally no in-process - target-model path. + soon as that call completes. Negotiated capture adapters register the + currently ready leases together per endpoint, then generate and publish + each lease independently. Prepared leases occupy bounded capture slots; + negotiated output byte reservations include all unfinished server writes + in the global feature-store hard cap until publication or exact cleanup. + There is intentionally no in-process target-model path. The producer has no training ledger or local ref queue; the consumer owns deduplication and durable acknowledgements. Returns ``(workers, drive_producer)``: @@ -1081,10 +1092,12 @@ def drive_producer(max_rounds: int = 1_000_000, should_stop=None) -> int: "accounted_consumed": channel.consumed_remote(), "resident_bytes": 0, "published_bytes": 0, + "reserved_bytes": 0, } published_sizes = deque() last_publish_log = {"t": time.perf_counter()} dead: dict = {} # worker_id -> last failure reason + inactive: dict = {} # explicitly stopped/draining workers # Set once the feeder will ingest nothing more (plan exhausted, stop, # or failure); only then may an empty pool end a worker. ingest_closed = threading.Event() @@ -1144,10 +1157,11 @@ def resident_bytes() -> int: with publish_lock: return reconcile_consumed_locked() - def publish_refs(w, refs, run_once_start: float) -> None: - if not refs: - return + def publish_refs(w, refs, run_once_start: float, reservation: int = 0) -> None: with publish_lock: + if not refs: + state["reserved_bytes"] -= reservation + return current_bytes = reconcile_consumed_locked() ref_sizes = [max(0, int(ref.estimated_bytes or 0)) for ref in refs] projected_bytes = current_bytes + sum(ref_sizes) @@ -1158,8 +1172,8 @@ def publish_refs(w, refs, run_once_start: float) -> None: cleanup_errors = [] for ref in refs: try: - feature_store.abort( - ref.sample_id, + abort_feature_ref( + feature_store, ref, reason="producer-resident-byte-hard-cap", ) except Exception as exc: @@ -1174,11 +1188,18 @@ def publish_refs(w, refs, run_once_start: float) -> None: if cleanup_errors: message += f"; cleanup errors={cleanup_errors}" raise MemoryError(message) - _publish_refs_with_cleanup( - channel=channel, - feature_store=feature_store, - refs=refs, - ) + try: + _publish_refs_with_cleanup( + channel=channel, + feature_store=feature_store, + refs=refs, + ) + except BaseException: + # A possibly visible prefix still owns its original credit. + # Stop peer admissions before releasing the shared lock. + abort.set() + raise + state["reserved_bytes"] -= reservation if channel.consumed_state_remote()[1] is None: published_sizes.extend(ref_sizes) state["published_bytes"] += sum(ref_sizes) @@ -1214,8 +1235,8 @@ def abort_unpublished(futures) -> None: continue for ref in refs: try: - feature_store.abort( - ref.sample_id, + abort_feature_ref( + feature_store, ref, reason="producer-driver-failed-before-publication", ) except Exception: @@ -1223,7 +1244,7 @@ def abort_unpublished(futures) -> None: "failed to abort unpublished ref %s", ref.sample_id ) - def run_worker(w) -> None: + def run_worker_legacy(w) -> None: failures = 0 submitted = 0 accepting = True @@ -1361,6 +1382,305 @@ def run_worker(w) -> None: abort_unpublished(futures) raise + def run_worker(w) -> None: + source = w.feature_source + if getattr(source, "_capture_protocol", None) is None: + return run_worker_legacy(w) + + # The endpoint's atomic reservation bound also limits each lease. + # Discover it before taking prompts so even a lease configured above + # the server capacity can be fulfilled without returning/failing it. + capacity_fn = getattr(source, "registration_capacity", None) + registration_capacity = capacity_fn() if callable(capacity_fn) else None + endpoint_lease = (min(worker_lease, registration_capacity) + if registration_capacity is not None else worker_lease) + + # One endpoint's network registration has its own executor. The + # coordinator only leases, reserves credits and harvests futures; + # an in-flight begin never blocks independent ref publication. + records = [] + futures = {} + preparing = registering = cleaning = None + failures = submitted = 0 + accepting = True + backpressure_started = None + capacity_blocked = False + next_registration_at = 0.0 + from specforge.inference.capture_protocol import CaptureCapacityDeclined + + def release_credit(record): + with publish_lock: + state["reserved_bytes"] -= record["credit"] + record["credit"] = 0 + + def reserve(record): + if record["credit"]: + return True # Same leased attempt retains its original credit. + size = record["prepared"]["estimated_bytes"] + if size is None or feature_store_max_resident_bytes is None: + return True + with publish_lock: + if abort.is_set(): + return False + current = reconcile_consumed_locked() + if size > feature_store_max_resident_bytes: + raise MemoryError( + "producer feature-store hard cap exceeded: " + f"sample reservation={size} > limit={feature_store_max_resident_bytes} bytes" + ) + if current + state["reserved_bytes"] + size > feature_store_max_resident_bytes: + return False + state["reserved_bytes"] += size + record["credit"] = size + return True + + def failed(exc): + nonlocal failures, accepting + failures += 1 + if failures >= max_worker_failures: + accepting = False + dead[w.worker_id] = str(exc) + + def clean_batch(group): + source.discard_prepared_batches([r["prepared"] for r in group]) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=producer_concurrency, + thread_name_prefix=f"capture-{w.worker_id}", + ) as executor, concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix=f"prepare-{w.worker_id}", + ) as preparation, concurrent.futures.ThreadPoolExecutor( + max_workers=1, thread_name_prefix=f"register-{w.worker_id}", + ) as registration: + try: + while True: + health_state = w.health()["state"] + if health_state in ("stopped", "draining"): + accepting = False + inactive[w.worker_id] = health_state + if abort.is_set() or (should_stop is not None and should_stop()): + accepting = False + # Harvest generates first, including while a different + # registration or uncertain-attempt cleanup is blocked. + for future in list(futures): + if not future.done(): + continue + record = futures.pop(future) + # A cancelled queued future never entered run_leased. + record["settled"] = not future.cancelled() + try: + refs = future.result() + except Exception as exc: + record.update(stage="cleanup", error=exc) + failed(exc) + else: + failures = 0 + credit = record["credit"] + missing = source.unreturned_prepared(record["prepared"], refs) + if missing is not None: + # Per-task errors settle controller leases but + # can still own server reservations or writes. + # Publish successes now; retain only failed + # identities and their credits until drained. + failed_credit = missing["estimated_bytes"] if credit else 0 + record.update(prepared=missing, tasks=missing["tasks"], + credit=failed_credit, stage="cleanup") + credit -= failed_credit + else: + record["credit"] = 0 + records.remove(record) + publish_refs(w, refs, record["start"], credit) + + if cleaning is not None and cleaning[0].done(): + future, group = cleaning + future.result() # Failure retains original ownership and aborts the driver. + for record in group: + release_credit(record) + if not record["settled"]: + w.fail_leased(record["tasks"], str(record["error"]), + unhealthy=not record.get("cancelled", False)) + records.remove(record) + cleaning = None + + if registering is not None and registering[0].done(): + future, group = registering + registering = None + try: + future.result() + except CaptureCapacityDeclined: + capacity_blocked = True + next_registration_at = time.monotonic() + backpressure_poll_s + for record in group: + record["stage"] = "pending" + except Exception as exc: + failed(exc) + for record in group: + record.update(stage="cleanup", error=exc) + else: + capacity_blocked = False + for record in group: + if (not accepting or abort.is_set() + or w.health()["state"] in ("stopped", "draining") + or (should_stop is not None and should_stop())): + accepting = False + record.update(stage="cleanup", error="producer-stopped", cancelled=True) + continue + record["stage"] = "generate" + future = executor.submit( + w.run_leased, record["tasks"], + max_tasks=endpoint_lease, prepared=record["prepared"], + ) + futures[future] = record + + if preparing is not None and preparing[0].done(): + future, group = preparing + preparing = None + try: + prepared = future.result() + if len(prepared) != len(group): + raise RuntimeError( + "capture preparation returned " + f"{len(prepared)} groups for {len(group)} leases" + ) + except Exception as exc: + failed(exc) + for record in group: + w.fail_leased(record["tasks"], f"prepare_refs:{exc}") + records.remove(record) + else: + for record, item in zip(group, prepared): + record.update(stage="pending", prepared=item) + + if not accepting: + # An unregistered prepared lease may be waiting for + # consumer credit. Cooperative stop must not depend + # on another ACK arriving before it can retire. + for record in records: + if record["stage"] == "pending": + record.update(stage="cleanup", error="producer-stopped", cancelled=True) + + # Credits for unfinished or uncertain server writes stay + # reserved until the exact attempt is physically drained. + if cleaning is None and registering is None: + group = [r for r in records if r["stage"] == "cleanup"] + if group: + cleaning = (registration.submit(clean_batch, group), group) + + in_flight = channel.in_flight_remote() + current_bytes = resident_bytes() + paused = in_flight >= consumer_quantum and flow_control.should_pause( + in_flight_refs=in_flight, resident_bytes=current_bytes, + ) + if accepting and not paused and submitted < round_budget and preparing is None: + group = [] + while len(records) < producer_concurrency and submitted < round_budget: + tasks = w.lease_tasks(endpoint_lease) + if not tasks: + break + record = dict(tasks=tasks, start=time.perf_counter(), + stage="prepare", credit=0, settled=False) + records.append(record) + group.append(record) + submitted += 1 + if group: + preparing = (preparation.submit( + source.prepare_refs_batches, + [r["tasks"] for r in group], capture=w.capture, + ), group) + + # Coalesce a ready snapshot within the endpoint's atomic + # begin bound. Excess leases retain their place while + # completed siblings publish and free server capacity. + budget_wait = False + if (registering is None and cleaning is None + and time.monotonic() >= next_registration_at): + group = [] + group_tasks = 0 + for record in records: + if record["stage"] != "pending": + continue + task_count = len(record["tasks"]) + if (registration_capacity is not None and + group_tasks + task_count > registration_capacity): + break + if reserve(record): + group_tasks += task_count + record["stage"] = "register" + group.append(record) + else: + budget_wait = True + if group: + registering = (registration.submit( + source.register_prepared_batches, + [r["prepared"] for r in group], + ), group) + + if paused or budget_wait or capacity_blocked: + if backpressure_started is None: + backpressure_started = time.monotonic() + if (peer_wait_timeout_s is not None and + time.monotonic() - backpressure_started > peer_wait_timeout_s): + raise TimeoutError("producer backpressure timed out waiting for consumer progress") + else: + backpressure_started = None + + if not records: + if ingest_closed.is_set() and pool_drained(): + return + if not accepting or submitted >= round_budget: + return + waits = list(futures) + waits.extend(item[0] for item in (preparing, registering, cleaning) if item is not None) + if waits: + concurrent.futures.wait(waits, timeout=backpressure_poll_s, + return_when=concurrent.futures.FIRST_COMPLETED) + else: + sleep(backpressure_poll_s) + except BaseException: + abort.set() + # Executor submit can fail before a future is installed. + # Every prelease remains in records, so none can disappear + # between preparation, registration and independent generate. + for item in (preparing, registering, cleaning): + if item is not None: + try: + result = item[0].result() + if item is preparing: + for record, prepared in zip(item[1], result): + record["prepared"] = prepared + except BaseException: + pass + cleanup_errors = [] + for future, record in futures.items(): + try: + refs = future.result() + except BaseException: + pass + else: + for ref in refs: + try: + abort_feature_ref(feature_store, ref, + reason="producer-driver-failed-before-publication") + except BaseException as exc: + cleanup_errors.append(f"{type(exc).__name__}: {exc}") + record["settled"] = not future.cancelled() + for record in records: + cleaned = False + try: + if "prepared" in record: + clean_batch([record]) + cleaned = True + except BaseException as exc: + cleanup_errors.append(f"{type(exc).__name__}: {exc}") + finally: + if not record["settled"]: + w.fail_leased(record["tasks"], "producer-driver-failed") + if cleaned: + release_credit(record) + if cleanup_errors: + logger.error("producer exact-attempt cleanup remains pending: %s", cleanup_errors) + raise + last_ingest_log = {"t": None} def ingest_prompt_batch(epoch: int, batch_index: int, epoch_prompts) -> None: @@ -1488,6 +1808,11 @@ def run_worker_guarded(w) -> None: f"all rollout workers exited with {len(dead)} dropped " f"as dead and prompts remaining — dead workers: {dead}" ) + if inactive and not stopped and remaining: + raise RuntimeError( + "rollout workers stopped or draining with prompts remaining: " + f"{inactive}" + ) if st["prompts_failed"] and not stopped: raise RuntimeError( "producer finished with " @@ -1567,6 +1892,8 @@ def build_disagg_online_consumer( async_ack: Optional[bool] = None, consumer_dispatch: str = "cost_balanced", dispatch_num_anchors: int = 512, + ref_poll_s: float = 0.1, + distributor_poll_s: float = 0.05, ): """Consumer (trainer) side of an ONLINE disaggregated run. @@ -1744,6 +2071,7 @@ def build_disagg_online_consumer( skip_ids=skip_ids, requeued_ids=requeued_ids, idle_timeout_s=idle_timeout_s, + poll_s=distributor_poll_s, ) inbox_server_url = os.environ.get("DISAGG_INBOX_SERVER_URL") if inbox_server_url: @@ -1801,7 +2129,7 @@ def build_disagg_online_consumer( inbox = RemoteInboxChannel(inbox_server_url, dp_rank) else: inbox = InboxChannel(RefDistributor.inbox_path(inbox_dir, dp_rank)) - queue = StreamingRefQueue(inbox, idle_timeout_s=idle_timeout_s) + queue = StreamingRefQueue(inbox, idle_timeout_s=idle_timeout_s, poll_s=ref_poll_s) drain_state = {"attempted": False} fit_failure = {"error": None} diff --git a/specforge/launch_plan.py b/specforge/launch_plan.py index 436a84ae85..28d6b8a2ad 100644 --- a/specforge/launch_plan.py +++ b/specforge/launch_plan.py @@ -772,6 +772,22 @@ def _managed_local_services( argv.extend(cfg.model.sglang_extra_args) argv.extend(server.extra_args) gpu_put = server.gpu_put if server.gpu_put is not None else default_gpu_put + hs_env = {"SPECFORGE_HS_CAPTURE": "0", "SPECFORGE_HS_CACHE": "0"} + if deployment.hs_capture: + if gpu_put: + raise ValueError("HS capture supports host publication only; disable gpu_put") + if mooncake.arena_bytes is not None: + raise ValueError("HS capture does not support NVLink arena publication") + gpu_put = False + hs_env = { + "SPECFORGE_HS_CAPTURE": "1", + "SPECFORGE_HS_CACHE": "1" if deployment.hs_prefix_cache else "0", + **{ + "SPECFORGE_HS_" + name.upper(): str(getattr(deployment, "hs_" + name)) + for name in ("chunk_tokens", "cache_bytes", "pinned_bytes", + "output_bytes", "device_bytes", "max_workers", "attempt_capacity") + }, + } python_env: dict[str, Optional[str]] = {} if server.python_executable: python_env = {"PYTHONPATH": None, "VIRTUAL_ENV": None} @@ -789,7 +805,9 @@ def _managed_local_services( else {} ), **({"SPECFORGE_SPEC_CAPTURE": "1"} if server_capture is not None else {}), - "FLASHINFER_DISABLE_VERSION_CHECK": "1", + **hs_env, + # Installation/runtime compatibility is checked by the real guard. + "FLASHINFER_DISABLE_VERSION_CHECK": None, "MOONCAKE_GLOBAL_SEGMENT_SIZE": str(mooncake.global_segment_size_bytes), "MOONCAKE_LOCAL_BUFFER_SIZE": str(mooncake.local_buffer_size_bytes), **( diff --git a/specforge/lr_scheduler.py b/specforge/lr_scheduler.py index c375842bd0..b0db9b5cc0 100644 --- a/specforge/lr_scheduler.py +++ b/specforge/lr_scheduler.py @@ -1,3 +1,4 @@ +import math from warnings import warn from torch.optim.lr_scheduler import CosineAnnealingLR as _CosineAnnealingLR @@ -119,6 +120,40 @@ def __init__( super().__init__(optimizer, warmup_steps, base_scheduler, last_epoch=last_epoch) +class TorchSpecCosineWarmupLR(_LRScheduler): + """Opt-in match for the archived TorchSpec cosine schedule. + + With warmup, step zero is zero LR and warmup reaches peak at warmup_steps. + Without warmup, step zero is the peak. Cosine reaches eta_min at total_steps. + The default SpecForge path is unchanged. + """ + + def __init__( + self, optimizer, total_steps, warmup_steps=0, eta_min=0.0, last_epoch=-1 + ): + if not 0 <= warmup_steps < total_steps: + raise ValueError("warmup_steps must be in [0, total_steps)") + if eta_min < 0 or any(eta_min > g["lr"] for g in optimizer.param_groups): + raise ValueError("eta_min must be between zero and each peak LR") + self.total_steps = int(total_steps) + self.warmup_steps = int(warmup_steps) + self.eta_min = float(eta_min) + super().__init__(optimizer, last_epoch) + + def get_lr(self): + step = self.last_epoch + if self.warmup_steps and step <= self.warmup_steps: + return [peak * step / self.warmup_steps for peak in self.base_lrs] + progress = min( + 1.0, + max( + 0.0, (step - self.warmup_steps) / (self.total_steps - self.warmup_steps) + ), + ) + coeff = 0.5 * (math.cos(math.pi * progress) + 1.0) + return [self.eta_min + coeff * (peak - self.eta_min) for peak in self.base_lrs] + + class _FlatLR(_LRScheduler): """Keep every parameter group at its configured base learning rate.""" diff --git a/specforge/optimizer.py b/specforge/optimizer.py index 6fd4b6f467..4ebbabb52f 100644 --- a/specforge/optimizer.py +++ b/specforge/optimizer.py @@ -4,7 +4,11 @@ import torch import torch.distributed as dist -from specforge.lr_scheduler import ConstantWarmupLR, CosineAnnealingWarmupLR +from specforge.lr_scheduler import ( + ConstantWarmupLR, + CosineAnnealingWarmupLR, + TorchSpecCosineWarmupLR, +) from specforge.utils import print_on_rank0 logger = logging.getLogger(__name__) @@ -49,6 +53,7 @@ def __init__( warmup_ratio=0.015, lr_scheduler="cosine", offload_master=False, + min_lr=0.0, ): # defaults copied from EAGLE traineagle3 ds_config.json self.model = model @@ -84,6 +89,7 @@ def __init__( scheduler_types = { "constant": ConstantWarmupLR, "cosine": CosineAnnealingWarmupLR, + "torchspec_cosine": TorchSpecCosineWarmupLR, } if lr_scheduler not in scheduler_types: raise ValueError( @@ -91,10 +97,12 @@ def __init__( f"expected one of {sorted(scheduler_types)}" ) self.lr_scheduler_type = lr_scheduler + scheduler_kwargs = {} if lr_scheduler == "constant" else {"eta_min": min_lr} self.scheduler = scheduler_types[lr_scheduler]( self.optimizer, total_steps=total_steps, warmup_steps=int(warmup_ratio * total_steps), + **scheduler_kwargs, ) def configure_grad_norm_reduction( diff --git a/specforge/runtime/control_plane/controller.py b/specforge/runtime/control_plane/controller.py index 53a577f50c..c5cdb6dc57 100644 --- a/specforge/runtime/control_plane/controller.py +++ b/specforge/runtime/control_plane/controller.py @@ -22,6 +22,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import dataclasses import threading import uuid @@ -250,10 +252,12 @@ def reconcile_on_restart( # never have materialized refs that belonged to another DP # rank, so seed generation/key bookkeeping from the durable # committed ref before asking it to remove remote objects. + if ref is None: + raise RuntimeError("cannot reconcile released sample without its committed ref") adopt = getattr(feature_store, "adopt", None) - if callable(adopt) and ref is not None: + if callable(adopt): adopt(ref) - feature_store.abort(sample_id, reason="reconciled-released") + abort_feature_ref(feature_store, ref, reason="reconciled-released") continue if ref is not None: self.sample_queue.put([ref]) diff --git a/specforge/runtime/data_plane/feature_dataloader.py b/specforge/runtime/data_plane/feature_dataloader.py index a01186e960..a57e02b9bd 100644 --- a/specforge/runtime/data_plane/feature_dataloader.py +++ b/specforge/runtime/data_plane/feature_dataloader.py @@ -17,6 +17,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import os import queue as queue_module import threading @@ -350,7 +352,7 @@ def _settle_incomplete_queue_batch(self, refs: List[SampleRef]) -> None: adopt = getattr(self.store, "adopt", None) if callable(adopt): adopt(ref) - self.store.abort(ref.sample_id, reason=reason) + abort_feature_ref(self.store, ref, reason=reason) except Exception as exc: # settle the full tail, then fail loudly cleanup_errors.append(f"{ref.sample_id}: {exc}") if cleanup_errors: diff --git a/specforge/runtime/data_plane/feature_store.py b/specforge/runtime/data_plane/feature_store.py index 372cc969db..588608e548 100644 --- a/specforge/runtime/data_plane/feature_store.py +++ b/specforge/runtime/data_plane/feature_store.py @@ -166,6 +166,15 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: return {"force_freed": 0, "force_freed_bytes": 0, "release_pending": 0} +def abort_feature_ref(store, ref: SampleRef, *, reason: str) -> None: + """Abort the exact ref held by a delayed callback, with legacy fallback.""" + abort_ref = getattr(store, "abort_ref", None) + if callable(abort_ref): + abort_ref(ref, reason=reason) + else: + store.abort(ref.sample_id, reason=reason) + + def drain_feature_store_removals( store: FeatureStore, *, diff --git a/specforge/runtime/data_plane/mooncake_store.py b/specforge/runtime/data_plane/mooncake_store.py index ee1d478ac1..4c462e0d51 100644 --- a/specforge/runtime/data_plane/mooncake_store.py +++ b/specforge/runtime/data_plane/mooncake_store.py @@ -58,6 +58,9 @@ from __future__ import annotations +from contextlib import contextmanager, ExitStack +from dataclasses import dataclass + import logging import os import threading @@ -538,6 +541,18 @@ def health(self) -> Dict[str, Any]: } +@dataclass(frozen=True) +class _CaptureBeginIntent: + """Stable retry identity and its batch-scoped singleflight owner.""" + protocol: Any + capability: dict + payload: dict + tokens: tuple + attempt_id: str + names: tuple + flight: Any + + class MooncakeFeatureStore(FeatureStore): """A disaggregated :class:`FeatureStore` backed by the Mooncake store. @@ -589,6 +604,9 @@ def __init__( self.store_id = store_id or uuid.uuid4().hex[:8] self._owns_store = store is None self._closed = False + self._closing = False + self._active_get_calls = 0 + self._close_lock = threading.Lock() if store is None: kw = dict(_MOONCAKE_SETUP_DEFAULTS) kw.update(setup_kwargs or {}) @@ -671,6 +689,17 @@ def __init__( # can reclaim those provisional objects; a successful adopt clears it. self._external_provisional: Dict[Tuple[str, int], List[str]] = {} self._active_leases: Dict[str, FeatureHandle] = {} + self._capture_fences = {} + self._capture_quarantined_registrations = set() + self._capture_handle_tokens = {} + self._capture_ack_pending = {} + self._capture_pending_begin = {} + self._terminal_cleanup_authority = None + self._terminal_cleanup_lock = threading.RLock() + self._capture_cleanup_local = threading.local() + self._capture_read_local = threading.local() + self._capture_read_intents = {} + self._capture_reads = {} # Samples whose remote remove() failed. gc() performs bounded # steady-state retries; lifecycle drain either removes them or raises. self._release_pending: Dict[str, int] = {} @@ -682,16 +711,356 @@ def __init__( # frees within a run (empty in retain_on_release/offline mode); a durable # shared index would own this in the online multi-node follow-up. self._freed: set = set() + self._freed_capture_attempts: set = set() self._lock = threading.RLock() self._counter = 0 self._gen_counter = 0 self._stats = {"force_freed": 0, "force_freed_bytes": 0} + def _begin_capture_read(self, ref): + fence = self._capture_from_ref(ref) + if fence is None: + return None + owner = uuid.uuid4().hex + intent = {"fence": fence, "owner": owner, "token": None, "unsafe_fetch": False, + "handle": None, "release_requested": False, "active": True} + with self._lock: + if self._capture_fences.get(ref.sample_id) is not fence or self._capture_ref_freed_locked(ref.sample_id, fence.wire): + raise KeyError('capture read ownership was released') + self._capture_read_intents[owner] = intent + # Keep intent before HTTP: lost acquire replies recover the same owner. + try: + with fence.lock: + intent["token"] = owner if fence.transferred else fence.acquire_read(owner) + except BaseException as exc: + from specforge.inference.capture_protocol import CaptureControlRejected + intent["active"] = False + if isinstance(exc, CaptureControlRejected): + with self._lock: + self._capture_read_intents.pop(owner, None) + raise + return intent + + def _release_capture_read(self, intent): + if intent["unsafe_fetch"]: + raise RuntimeError("capture receive drain unknown; read ownership retained") + fence = intent["fence"] + with fence.lock: + if not fence.transferred: + if intent["token"] is None: + intent["token"] = fence.acquire_read(intent["owner"]) + fence.release_read(intent["owner"], intent["token"]) + with self._lock: + self._capture_read_intents.pop(intent["owner"], None) + handle = intent["handle"] + if handle is not None: + self._capture_reads.pop(handle.lease_token, None) + if intent["release_requested"]: + self._active_leases.pop(handle.lease_token, None) + self._capture_handle_tokens.pop(handle.lease_token, None) + sid = handle.sample_id + if (not self.retain_on_release and self._capture_fences.get(sid) is fence + and not self._still_leased_locked(sid, handle.generation) + and not any(other["fence"] is fence for other in self._capture_read_intents.values())): + self._mark_freed_locked(sid, handle.generation) + self._release_pending.setdefault(sid, 0) + + def _recover_capture_reads(self, *, require_drained=False): + from specforge.inference.capture_protocol import CaptureControlRejected + with self._lock: + intents = list(self._capture_read_intents.values()) + failures = [] + for intent in intents: + if intent.get("active"): + continue + if intent["unsafe_fetch"]: + failures.append(intent["owner"] + ": receive drain unknown") + continue + if intent["handle"] is None or intent["release_requested"]: + try: + self._release_capture_read(intent) + except CaptureControlRejected as exc: + if intent["token"] is None: + # No native read started; authoritative refusal proves + # this owner has no active permit. Retire this intent only. + with self._lock: + self._capture_read_intents.pop(intent["owner"], None) + else: + intent["recovery_error"] = str(exc) + failures.append(intent["owner"] + ": " + str(exc)) + except Exception as exc: + intent["recovery_error"] = str(exc) + failures.append(intent["owner"] + ": " + str(exc)) + if require_drained and failures: + raise RuntimeError("capture read ownership not drained; permits retained: " + repr(failures)) + + def begin_capture_attempts(self, protocol, capability, requests): + """Reserve one batch while unrelated identities contact servers in parallel. + + A flight belongs to these immutable intents, not to the shared store. + Recovery can wait for this exact batch without serializing other servers. + """ + flight = threading.RLock() + # Prepare long token snapshots outside the metadata critical section. + intents = [(str(payload["sample_id"]), _CaptureBeginIntent( + protocol, dict(capability), dict(payload), tuple(tokens), attempt_id, + tuple(names), flight)) for payload, tokens, attempt_id, names in requests] + ids = [sid for sid, _ in intents] + with flight: + with self._lock: + if len(set(ids)) != len(ids) or any(sid in self._capture_pending_begin for sid in ids): + raise RuntimeError("duplicate or pending capture begin intent") + if any(sid in self._capture_fences or sid in self._capture_ack_pending or sid in self._release_pending for sid in ids): + raise RuntimeError("prior capture attempt has not relinquished ownership") + self._capture_pending_begin.update(intents) + if not capability.get('begin_batch'): + return [self._recover_capture_begin(sid) for sid in ids] + from specforge.inference.capture_protocol import CaptureCapacityDeclined + try: + replies = protocol.begin_batch(capability, [(intent.payload, intent.tokens, intent.attempt_id) for _, intent in intents]) + except CaptureCapacityDeclined: + # The typed atomic refusal proves none of this batch acquired a + # server writer. Unknown/lost replies must retain their intents. + with self._lock: + for sid, intent in intents: + if self._capture_pending_begin.get(sid) is intent: + self._capture_pending_begin.pop(sid) + raise + for (sid, intent), wire in zip(intents, replies): + self._finish_capture_begin(sid, intent, wire) + return replies + + def begin_capture_attempt(self, protocol, capability, payload, tokens, attempt_id, feature_names): + return self.begin_capture_attempts(protocol, capability, [(payload, tokens, attempt_id, feature_names)])[0] + + def _finish_capture_begin(self, sid, intent, wire): + # Caller holds intent.flight. No waiter may retarget this operation to + # a replacement endpoint/epoch merely because it has the same sample id. + with self._lock: + if self._capture_pending_begin.get(sid) is not intent: + raise RuntimeError("capture begin intent changed before completion") + self.track_capture_fence(wire, intent.names, control_fn=intent.protocol.post) + self.track_external_attempt(sid, generation=1, feature_names=list(intent.names), + server_url=intent.protocol.url.removesuffix("/spec_capture/control")) + with self._lock: + if self._capture_pending_begin.get(sid) is intent: + self._capture_pending_begin.pop(sid, None) + + def _recover_capture_begin(self, sid, *, expected_intent=None): + with self._lock: + intent = self._capture_pending_begin.get(sid) + if expected_intent is not None and intent is not expected_intent: + return None + if intent is None: + return None # Fresh samples never wait for another endpoint's HTTP. + with intent.flight: + with self._lock: + if self._capture_pending_begin.get(sid) is not intent: + return None + wire = intent.protocol.begin(intent.capability, intent.payload, intent.tokens, intent.attempt_id) + self._finish_capture_begin(sid, intent, wire) + return wire + + @contextmanager + def _capture_cleanup_scope(self, sample_ids, *, expected_records=None): + # The initiating caller may already have crossed a network operation; + # carry its original identity rather than snapshotting a replacement. + with self._lock: + records = (dict(expected_records) if expected_records is not None else + {sid: self._capture_fences.get(sid) for sid in sorted(set(sample_ids))}) + local = self._capture_cleanup_local + previous = getattr(local, "records", {}) + depth = getattr(local, "depth", 0) + with ExitStack() as stack: + for sid, fence in records.items(): + if fence is None: + continue + stack.enter_context(fence.lock) + with self._lock: + still_current = self._capture_fences.get(sid) is fence + if still_current: + if fence.transferred: + with self._lock: + if any(intent['fence'] is fence for intent in self._capture_read_intents.values()): + raise RuntimeError("capture local readers or receive transfers have not drained") + try: + fence.acquire_cleanup() + except BaseException: + with self._lock: + if self._capture_fences.get(sid) is fence: + self._generation.setdefault(sid, int(fence.wire["gen"])) + self._sample_names.setdefault(sid, list(fence.names)) + self._sample_bytes.setdefault(sid, 0) + self._release_pending.setdefault(sid, 0) + raise + local.records = {**previous, **records} + local.depth = depth + 1 + try: + yield records + finally: + local.records, local.depth = previous, depth + if depth == 0: + self._flush_capture_acks(sample_ids=records) + + def _capture_cleanup_allows(self, sid): + records = getattr(self._capture_cleanup_local, "records", {}) + if sid not in records: + return self._capture_fences.get(sid) is None + return self._capture_fences.get(sid) is records[sid] + + def track_capture_fence(self, wire, feature_names, *, control_fn=None): + from specforge.inference.capture_protocol import WriterFence, http_control + sid = str(wire["sample_id"]) + if str(wire["store_id"]) != str(self.store_id) or int(wire["gen"]) != 1: + raise ValueError("capture fence store/generation mismatch") + with self._lock: + old = self._capture_fences.get(sid) + if old is not None: + if old.token != wire["reservation_token"]: + raise RuntimeError("capture sample still has an unclean attempt") + return old + if any(h.sample_id == sid for h in self._active_leases.values()): + raise RuntimeError("capture sample has active consumer leases") + fence = WriterFence(dict(wire), tuple(feature_names), post=control_fn or http_control) + if wire.get('publication_ownership') == 'attempt-v1': + fence.accept_transfer(wire) + self._capture_fences[sid] = fence + # Only a server-authorized new reservation can retire the old g1 tombstone. + self._freed.discard((sid, 1)) + return fence + + def _capture_from_ref(self, ref, *, validate_remote=False): + wire = ref.metadata.get("capture_fence") + if wire is None: + return + sid = ref.sample_id + if (wire.get("sample_id") != sid or str(wire.get("store_id")) != str(self.store_id) + or ref.metadata.get("generation") is None + or int(wire.get("gen", -1)) != int(ref.metadata["generation"])): + raise ValueError("capture ref sample/store/generation identity mismatch") + with self._lock: + old = self._capture_fences.get(sid) + if old is not None and old.token != wire["reservation_token"]: + raise KeyError("stale capture attempt ref") + if old is None and self._capture_ref_freed_locked(sid, wire): + raise KeyError("released capture attempt ref") + fence = self.track_capture_fence(wire, list(ref.feature_keys)) + if wire.get('publication_ownership') == 'attempt-v1': + fence.accept_transfer(wire) + if ref.feature_keys != wire['physical_keys']: + raise RuntimeError('capture ref physical keys mismatch') + if validate_remote and not fence.transferred: + status = fence.control("resolve", cancel=False) + if status.get("state") != "PUBLISHED" or not status.get("writer_drained"): + from specforge.inference.capture_protocol import CaptureControlRejected + raise CaptureControlRejected("cannot adopt an attempt that is not currently PUBLISHED") + # A returned complete SampleRef is emitted only after writer completion. + fence.writer_drained = True + fence.adopted = True + return fence + + def _resolve_capture_writer(self, sid, expected_token=None): + with self._lock: + fence = self._capture_fences.get(sid) + if fence is None: + return + if expected_token is not None and fence.token != expected_token: + return + with fence.lock: + fence.drain(cancel=True) + + def _flush_capture_acks(self, sample_ids=None): + if getattr(self._capture_cleanup_local, "depth", 0): + return + with self._lock: + pending = sorted((sid, fence) for sid, fence in self._capture_ack_pending.items() + if sample_ids is None or (sid in sample_ids and (not isinstance(sample_ids, dict) or sample_ids[sid] is fence))) + for sid, fence in pending: + with fence.lock: + fence.acknowledge() + with self._lock: + if self._capture_ack_pending.get(sid) is fence: + self._capture_ack_pending.pop(sid, None) + if self._capture_fences.get(sid) is fence: + self._capture_fences.pop(sid, None) + + def discard_capture_attempt(self, sample_id, *, server_epoch, attempt_id, + control_url, reservation_token=None): + """Retire only the original prepared writer, even after a peer retries. + + An unknown begin reply can be recovered only through its matching + immutable intent. The captured fence then stays the cleanup authority + through resolve, physical removal and acknowledgement. + """ + sid = str(sample_id) + with self._lock: + intent = self._capture_pending_begin.get(sid) + if intent is not None and not ( + intent.protocol.url == control_url + and intent.capability["server_epoch"] == server_epoch + and intent.attempt_id == attempt_id + ): + intent = None + if intent is not None: + self._recover_capture_begin(sid, expected_intent=intent) + with self._lock: + fence = self._capture_fences.get(sid) + if fence is None or any(fence.wire.get(name) != value for name, value in ( + ("server_epoch", server_epoch), ("attempt_id", attempt_id), + ("control_url", control_url), + )) or (reservation_token is not None and fence.token != reservation_token): + return + if fence.adopted and sid in self._generation and sid not in self._release_pending: + raise RuntimeError("cannot discard an adopted live capture attempt") + with self._capture_cleanup_scope([sid], expected_records={sid: fence}): + with self._lock: + if self._capture_fences.get(sid) is not fence: + return + if fence.adopted and sid in self._generation and sid not in self._release_pending: + raise RuntimeError("cannot discard an adopted live capture attempt") + self._abort_impl(sid, reason="capture-prepared-discard", attempt_token=fence.token) + self.drain_sample_removals([sid]) + with self._lock: + if (self._capture_fences.get(sid) is fence + or self._capture_ack_pending.get(sid) is fence): + raise RuntimeError("prepared capture cleanup has not drained") + + def prepare_capture_retry(self, sample_id): + sid = str(sample_id) + self._recover_capture_begin(sid) + with self._lock: + fence = self._capture_fences.get(sid) + if any(h.sample_id == sid for h in self._active_leases.values()): + raise RuntimeError("cannot replace capture with active consumer lease") + adopted_live = fence is not None and fence.adopted and sid in self._generation and sid not in self._release_pending + if adopted_live and fence.transferred: + raise RuntimeError("cannot replace an adopted live capture sample before ownership handoff") + if adopted_live: + with fence.lock: + state = fence.control("resolve", cancel=False) + if state.get("state") != "RETIRED": + raise RuntimeError("cannot replace an adopted live capture sample") + fence.remote_retired = fence.writer_drained = fence.removed = fence.acknowledged = True + if fence is not None: + self._resolve_capture_writer(sid, fence.token) + self.abort(sid, reason="capture-retry", attempt_token=fence.token) + self.drain_sample_removals([sid]) + self._flush_capture_acks() + with self._lock: + if sid in self._capture_fences or sid in self._capture_ack_pending or sid in self._release_pending: + raise RuntimeError("prior capture cleanup has not drained") + # -- keys -------------------------------------------------------------- def _tkey(self, sample_id: str, gen: int, name: str) -> str: # generation lives in the key (like SharedDirFeatureStore's filename gen): # a re-put writes a new-gen key set and removes the old, so a stale ref's # keys are gone -> get() raises (B5), no payload-carried gen needed. + fence = self._capture_fences.get(sample_id) + if fence is not None and fence.wire.get('object_key_layout') == 'attempt-v1': + from specforge.inference.capture_protocol import attempt_keys + if name not in fence.names: + raise KeyError("feature not owned by capture attempt") + return attempt_keys(fence.wire, (name,))[name] return f"{self.store_id}/{sample_id}/g{gen}/{name}" # -- store wrappers (status-code aware) -------------------------------- @@ -746,23 +1115,44 @@ def consumer_device(self) -> Optional[torch.device]: return torch.device("cuda", index) def close(self) -> None: + with self._close_lock: + self._close_impl() + + def _close_impl(self) -> None: """Release receive memory after loaders and removal drains have stopped. An injected/shared backend is never closed here. Its failed-transfer buffers stay quarantined because its owner may still be using transport. """ - if self._closed: - return + with self._lock: + if self._closed: + return + # This fence is installed atomically with read admission. It stays + # closed after failed shutdown; existing reads may drain and close + # may be retried, but no new native receive can begin. + self._closing = True + if self._active_get_calls or any(intent["active"] or (intent["handle"] is not None and not intent["release_requested"]) + for intent in self._capture_read_intents.values()): + raise RuntimeError("cannot close capture transport with live readers") if self._receive_pool is not None: self._receive_pool.close() if self._owns_store: - rc = self._store.close() - if rc is not None and int(rc) != 0: - raise RuntimeError(f"Mooncake transport close failed ({rc})") + if not getattr(self, "_transport_stopped", False): + rc = self._store.close() + if rc is not None and int(rc) != 0: + raise RuntimeError(f"Mooncake transport close failed ({rc})") + self._transport_stopped = True if self._receive_pool is not None: self._receive_pool.close(transport_stopped=True) self._quarantined_buffers.clear() self._quarantined_bytes = 0 + self._capture_quarantined_registrations.clear() + # Successful transport shutdown proves every quarantined read has + # physically drained. An injected/shared transport has no such proof. + with self._lock: + for intent in self._capture_read_intents.values(): + intent["unsafe_fetch"] = False + self._recover_capture_reads(require_drained=True) self._closed = True def _fetch_tensor(self, key: str, spec, device="cpu") -> torch.Tensor: @@ -774,6 +1164,10 @@ def _fetch_tensor(self, key: str, spec, device="cpu") -> torch.Tensor: try: return self._fetch_tensor_once(key, spec, device) except _MooncakeGetError as exc: + if getattr(self._capture_read_local, "intent", None) is not None: + # A successful retry cannot prove a prior transfer drained. + # Keep the exact read permission until transport shutdown. + raise if exc.status not in _RETRYABLE_GET_STATUSES or attempt + 1 == attempts: raise delay = _GET_RETRY_DELAYS_S[attempt] @@ -793,10 +1187,11 @@ def _fetch_tensor_once(self, key: str, spec, device) -> torch.Tensor: dst = _alloc_from_spec(spec) try: self._store_get_tensor(key, dst) - except _MooncakeGetError: - # Quarantine the final failed attempt too: propagating its - # exception must not free storage that a late transfer can use. - self._quarantine_buffer(dst) + except BaseException as exc: + # Native get_into can raise before confirming physical drain. + # A failed fenced read keeps both storage and registration. + if isinstance(exc, _MooncakeGetError) or getattr(self._capture_read_local, "intent", None) is not None: + self._quarantine_buffer(dst) raise return dst dtype = _TORCH_DTYPES.get(spec.dtype) @@ -861,14 +1256,21 @@ def _store_get_tensor(self, key: str, out: torch.Tensor) -> None: self._store.register_buffer(out.data_ptr(), nb) except Exception: # pragma: no cover - some builds auto-register pass + drained = False try: rc = self._store.get_into(key, out.data_ptr(), nb) + _check_get_result(key, rc, nb) + drained = True finally: - try: - self._store.unregister_buffer(out.data_ptr()) - except Exception: # pragma: no cover - pass - _check_get_result(key, rc, nb) + if drained or getattr(self._capture_read_local, "intent", None) is None: + try: + self._store.unregister_buffer(out.data_ptr()) + except Exception: # pragma: no cover + pass + else: + # Unregistering before an uncertain DMA completes is unsafe. + # The owned transport's successful close is the release proof. + self._capture_quarantined_registrations.add(out.data_ptr()) def _store_remove(self, key: str, *, force: bool = False) -> bool: """Best-effort physical free. Returns True once the key is gone. @@ -985,6 +1387,7 @@ def adopt(self, sample_ref: SampleRef) -> None: raise ValueError( f"cannot adopt {sample_ref.sample_id}: ref carries no generation" ) + self._capture_from_ref(sample_ref, validate_remote=True) self._locate(sample_ref) with self._lock: gen = int(gen) @@ -1024,8 +1427,31 @@ def track_external_attempt( ) self._external_provisional[(str(sample_id), int(generation))] = names - def discard_external_attempts( - self, *, reason: str = "unadopted-external-capture" + def discard_external_attempts(self, *, reason="unadopted-external-capture"): + self._recover_capture_reads(require_drained=True) + with self._lock: + pending_begins = list(self._capture_pending_begin) + for sid in pending_begins: + self._recover_capture_begin(sid) + with self._lock: + snapshot = [(identity, names, self._capture_fences.get(identity[0])) + for identity, names in self._external_provisional.items()] + discarded = 0 + legacy = [] + for (sid, gen), names, expected in snapshot: + if expected is None: + legacy.append(((sid, gen), names)) + continue + with expected.lock: + with self._lock: + if self._capture_fences.get(sid) is not expected or (sid, gen) not in self._external_provisional: + continue + self.abort(sid, reason=reason, attempt_token=expected.token) + discarded += 1 + return discarded + (self._discard_external_attempts_legacy(reason=reason, expected_attempts=legacy) if legacy else 0) + + def _discard_external_attempts_legacy( + self, *, reason: str = "unadopted-external-capture", expected_attempts=() ) -> int: """Abort server writes that never produced an adoptable response. @@ -1035,18 +1461,21 @@ def discard_external_attempts( shutdown path. Physical-remove failures remain visible to :meth:`drain_pending_removals`. """ - with self._lock: - attempts = list(self._external_provisional.items()) + attempts = expected_attempts errors: Dict[str, str] = {} discarded = 0 for (sample_id, generation), names in attempts: + with self._lock: + fence = self._capture_fences.get(sample_id) + if fence is not None: + continue with self._lock: attempt_key = (sample_id, generation) # A response can be adopted after the snapshot but before this # attempt is visited. adopt() removes the provisional entry # under the same lock, so never delete that now-live sample. - if attempt_key not in self._external_provisional: + if self._external_provisional.get(attempt_key) is not names or self._capture_fences.get(sample_id) is not None: continue current = self._generation.get(sample_id) if current is not None: @@ -1077,6 +1506,7 @@ def discard_external_attempts( continue self._external_provisional.pop(attempt_key, None) discarded += 1 + self._flush_capture_acks() if errors: raise RuntimeError( "could not discard all provisional external captures: " f"{errors}" @@ -1084,7 +1514,42 @@ def discard_external_attempts( return discarded # -- read -------------------------------------------------------------- - def get( + def get(self, sample_ref: SampleRef, *, device="cpu", names=None): + with self._lock: + if self._closing or self._closed: + raise RuntimeError("Mooncake feature store is closing or closed") + self._active_get_calls += 1 + try: + return self._get_with_capture_fence(sample_ref, device=device, names=names) + finally: + with self._lock: + self._active_get_calls -= 1 + + def _get_with_capture_fence(self, sample_ref: SampleRef, *, device="cpu", names=None): + self.auth.check(self._credential) + self._recover_capture_reads() + intent = self._begin_capture_read(sample_ref) + previous = getattr(self._capture_read_local, "intent", None) + self._capture_read_local.intent = intent + try: + result = self._get_impl(sample_ref, device=device, names=names) + except BaseException: + if intent is not None: + intent["active"] = False + if not intent["unsafe_fetch"]: + intent["release_requested"] = True + self._release_capture_read(intent) + raise + finally: + self._capture_read_local.intent = previous + if intent is not None: + with self._lock: + intent["handle"] = result[1] + self._capture_reads[result[1].lease_token] = intent + intent["active"] = False + return result + + def _get_impl( self, sample_ref: SampleRef, *, @@ -1092,10 +1557,11 @@ def get( names: Optional[List[str]] = None, ) -> Tuple[Dict[str, torch.Tensor], FeatureHandle]: self.auth.check(self._credential) + self._capture_from_ref(sample_ref) sid = sample_ref.sample_id ref_gen = sample_ref.metadata.get("generation") with self._lock: - if ref_gen is not None and (sid, int(ref_gen)) in self._freed: + if ref_gen is not None and self._capture_ref_freed_locked(sid, sample_ref.metadata.get("capture_fence") or {"gen": ref_gen}): # logically freed here; the remote bytes may still linger under # Mooncake's read-lease, but this ref must not resolve (B5) raise KeyError( @@ -1126,6 +1592,9 @@ def get( lease_token=f"{sid}:{self._counter}", ) self._active_leases[handle.lease_token] = handle + fence = self._capture_fences.get(sid) + if fence is not None: + self._capture_handle_tokens[handle.lease_token] = fence.token return out, handle def _locate(self, sample_ref: SampleRef) -> None: @@ -1168,7 +1637,13 @@ def _get_tensors( f"(freed, stale, or never written)" ) # fresh alloc (or a pooled slot copied out) -> clone-on-fetch for free (B5) - out[n] = self._fetch_tensor(key, spec, device) + try: + out[n] = self._fetch_tensor(key, spec, device) + except BaseException: + intent = getattr(self._capture_read_local, "intent", None) + if intent is not None: + intent["unsafe_fetch"] = True + raise return out, gen # -- lifetime ---------------------------------------------------------- @@ -1188,6 +1663,19 @@ def _try_physical_free( disable that probe because probing a still-live key would renew its lease and make every following remove fail again. """ + fence = self._capture_fences.get(sample_id) + expected = getattr(self._capture_cleanup_local, "records", {}).get(sample_id) + if expected is not None and fence is not expected: + return False + if fence is not None: + if expected is not fence: + return False + if fence.remote_retired: + return True # Other store retired old attempt: discard LOCAL metadata only. + if not fence.writer_drained or (not fence.transferred and not fence.cleanup_token): + return False + if fence is not None and fence.transferred and any(intent['fence'] is fence for intent in self._capture_read_intents.values()): + return False gen = self._generation.get(sample_id) if gen is None: return True # nothing tracked to remove (already freed) @@ -1203,6 +1691,14 @@ def _try_physical_free( def _free_bookkeeping_locked(self, sample_id: str) -> int: """Drop in-process tracking for a sample. Returns bytes accounted freed.""" + fence = self._capture_fences.get(sample_id) + if fence is not None: + if not self._capture_cleanup_allows(sample_id): + raise RuntimeError("stale cleanup cannot mutate replacement bookkeeping") + if not fence.writer_drained: + raise RuntimeError("free bookkeeping before capture writer drain") + fence.removed = True + self._capture_ack_pending[sample_id] = fence generation = self._generation.get(sample_id) nbytes = self._sample_bytes.pop(sample_id, 0) self._generation.pop(sample_id, None) @@ -1223,6 +1719,31 @@ def _still_leased_locked(self, sample_id: str, generation: Optional[int]) -> boo def release(self, handle: FeatureHandle, *, reason: str = "consumed") -> None: with self._lock: + intent = self._capture_reads.get(handle.lease_token) + expected = intent["fence"] if intent is not None else self._capture_fences.get(handle.sample_id) + if intent is None and handle.lease_token not in self._active_leases and expected is not None: + return + if intent is not None: + intent["release_requested"] = True + self._release_capture_read(intent) + if self.retain_on_release: + return + with self._lock: + if expected.transferred and any(other["fence"] is expected for other in self._capture_read_intents.values()): + return # The final local reader owns the eventual deletion. + with self._capture_cleanup_scope([handle.sample_id], expected_records={handle.sample_id: expected}): + return self._release_impl(handle, reason=reason) + + def _release_impl(self, handle: FeatureHandle, *, reason: str = "consumed") -> None: + with self._lock: + if not self._capture_cleanup_allows(handle.sample_id): + return + expected = self._capture_handle_tokens.pop(handle.lease_token, None) + fence = self._capture_fences.get(handle.sample_id) + if expected is not None and (fence is None or fence.token != expected): + self._active_leases.pop(handle.lease_token, None) + return + self._active_leases.pop(handle.lease_token, None) if self.retain_on_release: return # offline re-iterable set: keep for the next epoch @@ -1232,23 +1753,70 @@ def release(self, handle: FeatureHandle, *, reason: str = "consumed") -> None: return # stale lease -> no-op if self._still_leased_locked(sid, cur): return - self._freed.add((sid, handle.generation)) # immediate logical free + self._mark_freed_locked(sid, handle.generation) # immediate logical free if self._try_physical_free(sid): self._free_bookkeeping_locked(sid) else: # remote free deferred (lease) / failed -> gc() retries self._release_pending.setdefault(sid, 0) - def abort(self, sample_id: str, *, reason: str = "aborted") -> None: + self._flush_capture_acks() + + def abort_ref(self, ref: SampleRef, *, reason="aborted") -> None: + wire = ref.metadata.get("capture_fence") + if wire is not None: + if wire.get("sample_id") != ref.sample_id or str(wire.get("store_id")) != str(self.store_id) or not wire.get("reservation_token"): + raise ValueError("capture cleanup ref identity mismatch") + self.abort(ref.sample_id, reason=reason, attempt_token=wire["reservation_token"]) + return + # A legacy callback carries no authority over a later negotiated + # attempt. Keep the generation comparison and delete under one lock. + with self._lock: + if ref.sample_id in self._capture_fences: + return + generation = ref.metadata.get("generation") + if generation is None or self._generation.get(ref.sample_id) != int(generation): + return + self.abort(ref.sample_id, reason=reason) + + def abort(self, sample_id: str, *, reason: str = "aborted", attempt_token=None) -> None: + with self._lock: + current = self._capture_fences.get(sample_id) + if attempt_token is not None and (current is None or current.token != attempt_token): + return + with self._capture_cleanup_scope([sample_id], expected_records={sample_id: current}): + return self._abort_impl(sample_id, reason=reason, attempt_token=attempt_token) + + def _abort_impl(self, sample_id: str, *, reason: str = "aborted", attempt_token=None) -> None: with self._lock: + if not self._capture_cleanup_allows(sample_id): + return + fence = self._capture_fences.get(sample_id) + if attempt_token is not None and (fence is None or fence.token != attempt_token): + return + if fence is not None: + self._resolve_capture_writer(sample_id, attempt_token) + with self._lock: + if fence is not None: + if self._capture_fences.get(sample_id) is not fence: + return + if any(h.sample_id == sample_id for h in self._active_leases.values()): + raise RuntimeError("cannot abort capture with active consumer lease") + # Lost responses have no adopted generation, but still own exact keys. + self._generation.setdefault(sample_id, int(fence.wire["gen"])) + self._sample_names.setdefault(sample_id, list(fence.names)) + self._sample_bytes.setdefault(sample_id, 0) + gen = self._generation.get(sample_id) if gen is not None: - self._freed.add((sample_id, gen)) # immediate logical free + self._mark_freed_locked(sample_id, gen) # immediate logical free if self._try_physical_free(sample_id): self._free_bookkeeping_locked(sample_id) else: self._release_pending.setdefault(sample_id, 0) + self._flush_capture_acks() + def drain_sample_removals( self, sample_ids: List[str], @@ -1262,14 +1830,23 @@ def drain_sample_removals( Other pending samples may belong to prefetched, not-yet-durable batches and must remain available for crash replay. """ - return self._drain_removals( + result = self._drain_removals( sample_ids=sample_ids, max_attempts=max_attempts, retry_interval_s=retry_interval_s, sleep=sleep, ) + self._flush_capture_acks() + return result + def retry_sample_removals(self, sample_ids: List[str]) -> Dict[str, Any]: + with self._lock: + targets = [sid for sid in sample_ids if sid in self._release_pending or sid in self._capture_ack_pending] + with self._capture_cleanup_scope(targets): + return self._retry_sample_removals_impl(sample_ids) + + def _retry_sample_removals_impl(self, sample_ids: List[str]) -> Dict[str, Any]: """Try selected removals once, without probing or sleeping. The optimizer path calls this only for samples made durable at an @@ -1286,6 +1863,8 @@ def retry_sample_removals(self, sample_ids: List[str]) -> Dict[str, Any]: if sample_id in target_ids ] for sample_id in pending: + if not self._capture_cleanup_allows(sample_id): + continue physically_removed = self._try_physical_free( sample_id, force=True, @@ -1307,6 +1886,7 @@ def retry_sample_removals(self, sample_ids: List[str]) -> Dict[str, Any]: for sample_id in self._release_pending if sample_id in target_ids ] + self._flush_capture_acks() return { "removed": removed, "removed_bytes": removed_bytes, @@ -1331,14 +1911,24 @@ def drain_pending_removals( ``sleep`` is injectable so protocol tests can advance a fake lease clock without wall-clock delays. """ - return self._drain_removals( + result = self._drain_removals( sample_ids=None, max_attempts=max_attempts, retry_interval_s=retry_interval_s, sleep=sleep, ) - def _drain_removals( + self._flush_capture_acks() + return result + + def _drain_removals(self, *, sample_ids, **kwargs): + self._recover_capture_reads(require_drained=True) + with self._lock: + targets = [sid for sid in set(self._release_pending) | set(self._capture_ack_pending) if sample_ids is None or sid in sample_ids] + with self._capture_cleanup_scope(targets): + return self._drain_removals_impl(sample_ids=sample_ids, **kwargs) + + def _drain_removals_impl( self, *, sample_ids: Optional[List[str]], @@ -1351,6 +1941,11 @@ def _drain_removals( if retry_interval_s < 0: raise ValueError("retry_interval_s must be >= 0") target_ids = None if sample_ids is None else set(sample_ids) + with self._lock: + fenced = [sid for sid in self._release_pending if target_ids is None or sid in target_ids] + for sid in fenced: + self._resolve_capture_writer(sid) + self._flush_capture_acks() removed = removed_bytes = 0 last_errors: Dict[str, str] = {} attempts_run = 0 @@ -1371,6 +1966,8 @@ def _drain_removals( } final_attempt = attempt + 1 == max_attempts for sample_id in pending: + if not self._capture_cleanup_allows(sample_id): + continue try: physically_removed = self._try_physical_free( sample_id, @@ -1430,7 +2027,127 @@ def _drain_removals( f"{preview}{detail}" ) + def _capture_ref_freed_locked(self, sid, wire): + if wire.get('object_key_layout') == 'attempt-v1': + return (sid, wire.get('reservation_token')) in self._freed_capture_attempts + return (sid, int(wire['gen'])) in self._freed + + def _mark_freed_locked(self, sid, generation): + self._freed.add((sid, generation)) + fence = self._capture_fences.get(sid) + if fence is not None and fence.wire.get('object_key_layout') == 'attempt-v1': + self._freed_capture_attempts.add((sid, fence.token)) + + def _published_capture_locked(self, sample_id): + fence = self._capture_fences.get(sample_id) + return fence is not None and fence.transferred + + def relinquish_published(self, ref): + """Disown a possibly visible ref after the streaming publisher handoff. + + This drops producer metadata only. The durable ref and its designated + consumer/optimizer authority now own deletion of the immutable objects. + """ + wire = ref.metadata.get('capture_fence') or {} + if wire.get('publication_ownership') != 'attempt-v1': + return + sid = ref.sample_id + with self._lock: + fence = self._capture_fences.get(sid) + if fence is None: + return + with fence.lock, self._lock: + if self._capture_fences.get(sid) is not fence or fence.token != wire.get('reservation_token'): + return + if not fence.transferred or any(h.sample_id == sid for h in self._active_leases.values()) or any(intent['fence'] is fence for intent in self._capture_read_intents.values()): + raise RuntimeError('cannot relinquish capture with local reader ownership') + self._mark_freed_locked(sid, int(wire['gen'])) + for mapping in (self._generation, self._sample_names, self._sample_bytes, self._put_time, + self._release_pending, self._capture_ack_pending, self._capture_fences): + mapping.pop(sid, None) + self._external_provisional.pop((sid, int(wire['gen'])), None) + + def _validate_terminal_capture_ref(self, ref, wire): + """Pure validation must precede even an already-cleaned fast return.""" + from specforge.inference.capture_protocol import WriterFence + + if (wire.get("protocol_version") != 1 or wire.get("sample_id") != ref.sample_id + or str(wire.get("store_id")) != str(self.store_id) + or int(wire.get("gen", -1)) != 1 + or ref.metadata.get("generation") is None + or int(ref.metadata["generation"]) != int(wire["gen"])): + raise ValueError("terminal capture ref sample/store/generation identity mismatch") + if ref.feature_store_uri != f"mooncake://{self.store_id}/{ref.sample_id}": + raise ValueError("terminal capture ref canonical URI identity mismatch") + proof = WriterFence(dict(wire), tuple(ref.feature_keys)) + if not proof.accept_transfer(wire) or ref.feature_keys != wire.get("physical_keys"): + raise ValueError("terminal capture ref certificate/physical keys mismatch") + + def cleanup_terminal_refs(self, refs, *, consumer_done=False, producer_closed=False): + """Reclaim exact published attempts after both sides have finished. + + Normal handoff tombstones stay sealed. A separate metadata authority + may adopt immutable refs only after successful consumer drain and the + producer's joined-worker EOF. Failure/timeout alone proves neither. + The authority borrows this native backend and must never close it. + """ + refs = tuple(refs) + errors = [] + cleaned = 0 + with self._terminal_cleanup_lock: + for ref in refs: + wire = ref.metadata.get("capture_fence") or {} + immutable = (wire.get("object_key_layout") == "attempt-v1" or + wire.get("publication_ownership") == "attempt-v1") + try: + if not immutable: + # Legacy cleanup retains its existing remote reader / + # writer permits, even during an abnormal shutdown. + self.abort_ref(ref, reason="online-attempt-finished") + cleaned += 1 + continue + if consumer_done is not True or producer_closed is not True: + raise RuntimeError("terminal ownership unresolved: successful consumer_done and closed producer are required") + self._validate_terminal_capture_ref(ref, wire) + authority = self._terminal_cleanup_authority + if authority is None: + authority = MooncakeFeatureStore(store_id=self.store_id, store=self._store, + auth=self.auth, credential=self._credential, + max_release_attempts=self.max_release_attempts) + self._terminal_cleanup_authority = authority + token = wire.get("reservation_token") + with authority._lock: + already_cleaned = ((ref.sample_id, token) in authority._freed_capture_attempts + and ref.sample_id not in authority._capture_fences) + if not already_cleaned: + authority.adopt(ref) # Strict certificate + exact key validation. + authority.abort_ref(ref, reason="online-attempt-finished") + authority.drain_sample_removals([ref.sample_id]) + cleaned += 1 + except Exception as exc: + errors.append(f"{ref.sample_id}/{wire.get('reservation_token')}: {type(exc).__name__}: {exc}") + authority = self._terminal_cleanup_authority + if authority is not None and consumer_done is True and producer_closed is True: + try: + authority.drain_pending_removals() + except Exception as exc: + errors.append(f"terminal exact-key removal drain: {type(exc).__name__}: {exc}") + if errors: + raise RuntimeError("terminal published-ref cleanup unresolved: " + repr(errors)) + return {"refs_cleaned": cleaned} + def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: + self._recover_capture_reads() + current = self._clock() if now is None else now + with self._lock: + targets = set(self._release_pending) | set(self._capture_ack_pending) + if self.max_hold_age_s is not None: + targets.update(sid for sid, start in self._put_time.items() + if current-start > self.max_hold_age_s and not self._published_capture_locked(sid) and not self._still_leased_locked(sid, self._generation.get(sid))) + with self._capture_cleanup_scope(targets): + return self._gc_impl(now=current) + + def _gc_impl(self, *, now: Optional[float] = None) -> Dict[str, int]: now = self._clock() if now is None else now freed = freed_bytes = 0 with self._lock: @@ -1440,9 +2157,12 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: sid for sid, t in list(self._put_time.items()) if now - t > self.max_hold_age_s + and not self._published_capture_locked(sid) and not self._still_leased_locked(sid, self._generation.get(sid)) ] for sid in stale: + if not self._capture_cleanup_allows(sid): + continue if self._try_physical_free(sid, confirm_absent_on_failure=False): freed_bytes += self._free_bookkeeping_locked(sid) freed += 1 @@ -1451,6 +2171,8 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: # Reconcile release-pending without an exists probe: is_exist grants # a read lease that would make the next remove fail (-706). for sid in list(self._release_pending): + if not self._capture_cleanup_allows(sid): + continue if self._release_pending[sid] >= self.max_release_attempts: # Keep the physical key metadata and surface the pending # sample. Lifecycle drain owns the final bounded retry and @@ -1465,6 +2187,7 @@ def gc(self, *, now: Optional[float] = None) -> Dict[str, int]: self._release_pending[sid] = attempts self._stats["force_freed"] += freed self._stats["force_freed_bytes"] += freed_bytes + self._flush_capture_acks() return { "force_freed": freed, "force_freed_bytes": freed_bytes, @@ -1486,6 +2209,14 @@ def health(self) -> Dict[str, Any]: "backend": "mooncake", "resident_samples": len(self._generation), "provisional_external": len(self._external_provisional), + "capture_pending_begin": len(self._capture_pending_begin), + "capture_fences": len(self._capture_fences), + "capture_ack_pending": len(self._capture_ack_pending), + "capture_handle_attempts": len(self._capture_handle_tokens), + "capture_read_intents": len(self._capture_read_intents), + "capture_active_reads": len(self._capture_reads), + "capture_failed_reads": sum(bool(x["unsafe_fetch"]) for x in self._capture_read_intents.values()), + "capture_quarantined_registrations": len(self._capture_quarantined_registrations), "active_leases": len(self._active_leases), "resident_bytes": sum(self._sample_bytes.values()), "max_resident_bytes": self.max_resident_bytes, diff --git a/specforge/runtime/data_plane/ref_distributor.py b/specforge/runtime/data_plane/ref_distributor.py index 1ff78f5e04..005a0de7e3 100644 --- a/specforge/runtime/data_plane/ref_distributor.py +++ b/specforge/runtime/data_plane/ref_distributor.py @@ -44,6 +44,8 @@ from __future__ import annotations +from specforge.runtime.data_plane.feature_store import abort_feature_ref + import logging import os import threading @@ -473,7 +475,7 @@ def _finish(self) -> None: adopt = getattr(self.feature_store, "adopt", None) if callable(adopt): adopt(ref) - self.feature_store.abort(ref.sample_id, reason=reason) + abort_feature_ref(self.feature_store, ref, reason=reason) except Exception as exc: # best effort before the loud failure cleanup_errors.append(f"{ref.sample_id}: {exc}") self.source.mark_consumed( diff --git a/specforge/runtime/data_plane/streaming_ref_channel.py b/specforge/runtime/data_plane/streaming_ref_channel.py index 4f18b791d9..ff62de0702 100644 --- a/specforge/runtime/data_plane/streaming_ref_channel.py +++ b/specforge/runtime/data_plane/streaming_ref_channel.py @@ -225,6 +225,10 @@ def consumer_failure(self) -> Optional[str]: """Return the consumer failure reason, if one has been published.""" return self._read_sidecar(_CONSUMER_FAILED_SUFFIX) + def consumer_done(self) -> bool: + """Successful consumer drain proof; a failure sentinel always wins.""" + return os.path.exists(self.path + _CONSUMER_DONE_SUFFIX) and self.consumer_failure() is None + def consumer_stopped(self) -> bool: """Whether the remote consumer completed or failed.""" return os.path.exists(self.path + _CONSUMER_DONE_SUFFIX) or os.path.exists( diff --git a/specforge/training/assembly.py b/specforge/training/assembly.py index cd82c7b4eb..f37b83617f 100644 --- a/specforge/training/assembly.py +++ b/specforge/training/assembly.py @@ -265,6 +265,8 @@ def __call__(self, draft_module): return BF16Optimizer( draft_module, lr=t.learning_rate, + weight_decay=t.weight_decay, + min_lr=t.min_lr, max_grad_norm=t.max_grad_norm, warmup_ratio=t.warmup_ratio, lr_scheduler=t.lr_scheduler, @@ -356,6 +358,7 @@ def _prompt_cache_key( "path": source_path, "content_hash": content_hash, "max_length": cfg.data.max_length, + "overlength_policy": cfg.data.overlength_policy, "chat_template": cfg.data.chat_template, "tokenizer_chat_template_hash": _tokenizer_chat_template_hash(tokenizer), "is_preformatted": cfg.data.is_preformatted, @@ -410,6 +413,7 @@ def _prepare_prompts( tokenizer, chat_template=cfg.data.chat_template, max_length=cfg.data.max_length, + overlength_policy=cfg.data.overlength_policy, is_preformatted=cfg.data.is_preformatted, train_only_last_turn=cfg.data.train_only_last_turn, cache_dir=cfg.data.cache_dir, diff --git a/specforge/training/disaggregated.py b/specforge/training/disaggregated.py index 71f78b91ec..a2b2999c8e 100644 --- a/specforge/training/disaggregated.py +++ b/specforge/training/disaggregated.py @@ -604,6 +604,34 @@ def _producer_capture_metadata(cfg: Config, algorithm: AlgorithmRegistration): ) +def _cleanup_online_published_refs(store, channel): + """Run the existing terminal publication scan with exact-ref authority.""" + from specforge.runtime.data_plane.feature_store import abort_feature_ref + from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel + + reader = StreamingRefChannel(channel.path) + cleanup = getattr(store, "cleanup_terminal_refs", None) + errors = [] + while True: + refs = reader.poll(max_n=1024) + if not refs: + break + if callable(cleanup): + try: + cleanup(refs, consumer_done=channel.consumer_done(), + producer_closed=channel.is_closed()) + except Exception as exc: + errors.append(f"{type(exc).__name__}: {exc}") + else: + for ref in refs: + try: + abort_feature_ref(store, ref, reason="online-attempt-finished") + except Exception as exc: + errors.append(f"{ref.sample_id}: {type(exc).__name__}: {exc}") + if errors: + raise RuntimeError("terminal published-ref scan unresolved: " + repr(errors)) + + def _build_online( cfg: Config, *, @@ -703,6 +731,8 @@ def _build_online( request_input_adapter=input_adapter, target_model_version=cfg.model.target_model_path, server_capture=_server_capture(cfg), + hs_capture=cfg.deployment.disaggregated.hs_capture, + hs_prefix_cache=cfg.deployment.disaggregated.hs_prefix_cache, ) for url in _server_urls(cfg) ] @@ -741,6 +771,7 @@ def _build_online( feature_store_max_resident_bytes=( cfg.runtime.feature_store_max_resident_bytes ), + backpressure_poll_s=cfg.runtime.producer_backpressure_poll_s, peer_wait_timeout_s=peer_wait_timeout_s, ) @@ -775,18 +806,7 @@ def produce() -> int: ) cleanup_errors = [] try: - reader = StreamingRefChannel(channel_path) - while True: - refs = reader.poll(max_n=1024) - if not refs: - break - for ref in refs: - try: - store.abort(ref.sample_id, reason="online-attempt-finished") - except Exception as exc: - cleanup_errors.append( - f"{ref.sample_id}: {type(exc).__name__}: {exc}" - ) + _cleanup_online_published_refs(store, channel) except Exception as exc: cleanup_errors.append( f"published-ref scan: {type(exc).__name__}: {exc}" @@ -859,6 +879,8 @@ def produce() -> int: async_ack=_consumer_async_ack(cfg), consumer_dispatch=cfg.runtime.consumer_dispatch, dispatch_num_anchors=cfg.training.num_anchors, + ref_poll_s=cfg.runtime.consumer_ref_poll_s, + distributor_poll_s=cfg.runtime.ref_distributor_poll_s, ) return TrainingRun(trainer=trainer, on_finally=store.close) diff --git a/tests/plana/test_runtime_polling.py b/tests/plana/test_runtime_polling.py new file mode 100644 index 0000000000..fec2b9b276 --- /dev/null +++ b/tests/plana/test_runtime_polling.py @@ -0,0 +1,169 @@ +"""CPU gates for configurable polling without changing flow-control bounds.""" +import os +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock + +from pydantic import ValidationError +import yaml + +from specforge.algorithms.builtin import builtin_algorithm_registry +from specforge.config import Config, load_config +from specforge.config.schema import RuntimeConfig +from specforge.launch import build_disagg_online_consumer +from specforge.runtime.control_plane.metadata_store import InMemoryMetadataStore +from specforge.runtime.data_plane.ref_distributor import RefDistributor +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from specforge.training.disaggregated import _build_online + + +ALGORITHM=builtin_algorithm_registry().resolve('dflash') +POLL_FIELDS=('producer_backpressure_poll_s','consumer_ref_poll_s','ref_distributor_poll_s') +DEFAULTS=(0.2,0.1,0.05) +TUNED=(0.011,0.013,0.017) +BOUNDS={'producer_lease':1,'producer_concurrency':1, + 'in_flight_high_watermark':4,'in_flight_low_watermark':2, + 'resident_high_watermark_bytes':512<<20,'resident_low_watermark_bytes':256<<20, + 'feature_store_max_resident_bytes':1<<30} + + +def payload(root,role,*,polling=None): + runtime=dict(BOUNDS) + if polling is not None:runtime.update(zip(POLL_FIELDS,polling)) + return {'model':{'target_model_path':'target','draft_model_config':'draft'}, + 'data':{'prompts_path':'prompts.jsonl','dataloader_num_workers':0}, + 'training':{'strategy':'dflash','role':role,'max_steps':1,'num_anchors':8, + 'seed':42,'prompt_seed':42},'runtime':runtime, + 'deployment':{'mode':'disaggregated','disaggregated':{ + 'control_dir':str(root),'backend':'mooncake','async_ack':False, + 'server_urls':['http://capture:30000']}}} + + +class PollingSchemaTests(unittest.TestCase): + def test_defaults_preserve_fixed_upstream_cadences_and_bounds(self): + runtime=RuntimeConfig() + self.assertEqual(tuple(getattr(runtime,name) for name in POLL_FIELDS),DEFAULTS) + self.assertEqual((runtime.in_flight_high_watermark,runtime.in_flight_low_watermark),(256,192)) + self.assertEqual(runtime.producer_concurrency,1) + + def test_positive_finite_values_do_not_change_any_other_runtime_field(self): + default=RuntimeConfig(**BOUNDS).model_dump() + tuned=RuntimeConfig(**BOUNDS,**dict(zip(POLL_FIELDS,TUNED))).model_dump() + for field,value in zip(POLL_FIELDS,TUNED): + self.assertEqual(tuned.pop(field),value) + default.pop(field) + self.assertEqual(tuned,default) + + def test_nonpositive_nonfinite_and_non_numeric_values_are_rejected(self): + for field in POLL_FIELDS: + for value in (0.,-0.,-0.01,float('nan'),float('inf'),float('-inf'),True,False,'invalid',None): + with self.subTest(field=field,value=value),self.assertRaises(ValidationError): + RuntimeConfig(**{field:value}) + + def test_actual_yaml_loader_and_dotted_overrides_revalidate_polling(self): + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'run.yaml' + path.write_text(yaml.safe_dump(payload(tmp,'consumer',polling=TUNED))) + cfg=load_config(str(path),['runtime.consumer_ref_poll_s=0.023']) + self.assertEqual(cfg.runtime.producer_backpressure_poll_s,TUNED[0]) + self.assertEqual(cfg.runtime.consumer_ref_poll_s,0.023) + self.assertEqual(cfg.runtime.ref_distributor_poll_s,TUNED[2]) + for field in POLL_FIELDS: + for value in ('0.0','-0.01','nan','inf','-inf','true'): + with self.subTest(field=field,value=value),self.assertRaises(ValidationError): + load_config(str(path),[f'runtime.{field}={value}']) + + +class RoleFactoryWiringTests(unittest.TestCase): + def test_producer_config_reaches_existing_builder_parameter_with_bounds_unchanged(self): + for polling,expected in ((None,DEFAULTS),(TUNED,TUNED)): + with self.subTest(polling=polling),tempfile.TemporaryDirectory() as tmp: + cfg=Config.model_validate(payload(tmp,'producer',polling=polling)) + with ( + mock.patch.dict(os.environ,{'DISAGG_REF_CHANNEL':str(Path(tmp)/'refs'), + 'DISAGG_RESIDENT_HIGH_WATERMARK_BYTES':'', + 'DISAGG_RESIDENT_LOW_WATERMARK_BYTES':''}), + mock.patch('specforge.runtime.data_plane.streaming_ref_channel.StreamingRefChannel'), + mock.patch('specforge.training.disaggregated._mooncake_store'), + mock.patch('specforge.training.assembly._load_input_tools'), + mock.patch('specforge.training.model_loading.resolve_draft_config'), + mock.patch('specforge.training.disaggregated._producer_capture_metadata',return_value=([1,2],8,32,32)), + mock.patch('specforge.inference.adapters.server_capture.SGLangServerCaptureAdapter'), + mock.patch('specforge.launch.build_disagg_online_producer',return_value=([],mock.Mock())) as build, + ): + _build_online(cfg,algorithm=ALGORITHM,build_model_bundle=mock.Mock(), + prepare_prompts=lambda *a,**k:[{'task_id':'p0','payload':{'input_ids':[1,2,3]}}], + optimizer_factory=mock.Mock(),logger=None) + args=build.call_args.kwargs + self.assertEqual(args['backpressure_poll_s'],expected[0]) + for name in ('producer_concurrency','in_flight_high_watermark','in_flight_low_watermark', + 'resident_high_watermark_bytes','resident_low_watermark_bytes', + 'feature_store_max_resident_bytes'): + self.assertEqual(args[name],BOUNDS[name]) + self.assertEqual(args['lease'],1) + self.assertEqual(args['prompt_seed'],42) + + def test_consumer_config_reaches_builder_without_changing_ack_or_math(self): + for polling,expected in ((None,DEFAULTS),(TUNED,TUNED)): + with self.subTest(polling=polling),tempfile.TemporaryDirectory() as tmp: + cfg=Config.model_validate(payload(tmp,'consumer',polling=polling)) + bundle=SimpleNamespace(model=object(),target_head=None,strategy_kwargs={}) + with ( + mock.patch.dict(os.environ,{'DISAGG_REF_CHANNEL':str(Path(tmp)/'refs')}), + mock.patch('specforge.runtime.data_plane.streaming_ref_channel.StreamingRefChannel'), + mock.patch('specforge.training.disaggregated._mooncake_store'), + mock.patch('specforge.launch.build_disagg_online_consumer',return_value=SimpleNamespace()) as build, + ): + _build_online(cfg,algorithm=ALGORITHM,build_model_bundle=lambda c:bundle, + prepare_prompts=mock.Mock(),optimizer_factory=mock.Mock(),logger=None) + args=build.call_args.kwargs + self.assertEqual(args['ref_poll_s'],expected[1]) + self.assertEqual(args['distributor_poll_s'],expected[2]) + self.assertIs(args['draft_model'],bundle.model) + self.assertEqual(args['async_ack'],False) + self.assertEqual(args['dispatch_num_anchors'],8) + self.assertEqual(args['dataloader_num_workers'],0) + self.assertEqual(args['max_steps'],1) + + +class EffectiveConsumerPollingTests(unittest.TestCase): + def build(self,root,**polling): + channel=StreamingRefChannel(str(root/'refs.jsonl')) + store=SimpleNamespace(retain_on_release=True) + captured={} + def assemble(**kwargs): + captured.update(kwargs) + return SimpleNamespace() + with ( + mock.patch('torch.distributed.is_initialized',return_value=False), + mock.patch.object(RefDistributor,'start',autospec=True,side_effect=lambda self:self) as start, + mock.patch('specforge.launch._assemble_trainer',side_effect=assemble), + ): + trainer=build_disagg_online_consumer(algorithm=ALGORITHM,feature_store=store,channel=channel, + draft_model=object(),optimizer_factory=object(),run_id='polling-test',output_dir=str(root/'out'), + collate_fn=lambda features:{},metadata_store=InMemoryMetadataStore(), + inbox_dir=str(root/'inboxes'),async_ack=False,idle_timeout_s=12.5,**polling) + self.assertEqual(start.call_count,1) + return trainer,captured + + def test_existing_direct_call_keeps_real_queue_and_distributor_defaults(self): + with tempfile.TemporaryDirectory() as tmp: + trainer,captured=self.build(Path(tmp)) + self.assertEqual(captured['ref_source']['queue'].poll_s,0.1) + self.assertEqual(trainer.ref_distributor.poll_s,0.05) + self.assertEqual(captured['ref_source']['queue'].idle_timeout_s,12.5) + self.assertEqual(trainer.ref_distributor.idle_timeout_s,12.5) + + def test_custom_builder_values_reach_real_constructor_instances(self): + with tempfile.TemporaryDirectory() as tmp: + trainer,captured=self.build(Path(tmp),ref_poll_s=TUNED[1],distributor_poll_s=TUNED[2]) + self.assertEqual(captured['ref_source']['queue'].poll_s,TUNED[1]) + self.assertEqual(trainer.ref_distributor.poll_s,TUNED[2]) + self.assertTrue(captured['ref_source']['defer_ack_until_durable']) + self.assertEqual(captured['async_ack'],False) + self.assertEqual(trainer.ref_distributor.refs_per_rank_step,1) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_data/test_prompt_length_filter.py b/tests/test_data/test_prompt_length_filter.py new file mode 100644 index 0000000000..e4c2f9719b --- /dev/null +++ b/tests/test_data/test_prompt_length_filter.py @@ -0,0 +1,121 @@ +"""Whole-record filtering at the native loader's public boundary.""" + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from pydantic import ValidationError + +from specforge.config.schema import DataConfig +from specforge.data.loss_mask import has_consecutive_supervised_tokens +from specforge.data.prompt_builder import prepare_prompt_tasks + + +class TestPromptLengthFilter(unittest.TestCase): + def load(self, records, **options): + report = {} + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp, "data.jsonl") + path.write_text("".join(json.dumps(r) + "\n" for r in records)) + with contextlib.redirect_stdout(io.StringIO()) as output: + prompts = prepare_prompt_tasks( + path, + tokenizer=None, + chat_template=None, + max_length=65536, + is_preformatted=False, + train_only_last_turn=False, + cache_dir=None, + cache_key=None, + num_proc=1, + min_loss_tokens=2, + loss_mask_filter=has_consecutive_supervised_tokens, + overlength_policy="drop", + filter_report=report, + **options, + ) + self.assertIn("[prompt-filter]", output.getvalue()) + return prompts, report + + def test_boundary_and_original_ids_masks_survive(self): + records = [ + dict(id=str(n), input_ids=list(range(n)), loss_mask=[0] * (n - 2) + [1, 1]) + for n in (65535, 65536, 65537) + ] + prompts, report = self.load(records) + self.assertEqual( + [p["payload"] for p in prompts], + [{k: r[k] for k in ("input_ids", "loss_mask")} for r in records[:2]], + ) + self.assertEqual(report["accepted"], 2) + self.assertEqual(report["rows_seen"], 3) + self.assertEqual(report["dropped_other"], 0) + self.assertEqual(report["dropped_overlength"], 1) + self.assertEqual(report["kept_input_tokens"], 131071) + self.assertEqual(report["kept_loss_tokens"], 4) + self.assertEqual(report["dropped_samples"][0]["id"], "65537") + + def test_limit_counts_accepted_after_filter(self): + records = [ + dict(id="long", input_ids=[1] * 65537, loss_mask=[1] * 65537), + dict(id="invalid_supervision", input_ids=[2, 3, 4], loss_mask=[1, 0, 1]), + dict(id="keep", input_ids=[[4, 5]], loss_mask=[[1, 1]]), + dict(id="unread", input_ids=[6, 7], loss_mask=[1, 1]), + ] + prompts, report = self.load(records, max_prompts=1) + self.assertEqual( + prompts, [{"payload": {"input_ids": [4, 5], "loss_mask": [1, 1]}}] + ) + self.assertEqual(report["rows_seen"], 3) + self.assertEqual(report["dropped_other"], 1) + self.assertEqual(report["dropped_overlength"], 1) + + def test_zero_and_none_limits_keep_all_accepted_records(self): + records = [ + dict(id="long", input_ids=[1] * 65537, loss_mask=[1] * 65537), + dict(id="first", input_ids=[2, 3], loss_mask=[1, 1]), + dict(id="second", input_ids=[4, 5, 6], loss_mask=[0, 1, 1]), + ] + for limit in (None, 0): + with self.subTest(limit=limit): + prompts, report = self.load(records, max_prompts=limit) + self.assertEqual( + [p["payload"]["input_ids"] for p in prompts], [[2, 3], [4, 5, 6]] + ) + self.assertEqual(report["rows_seen"], 3) + self.assertEqual(report["accepted"], 2) + self.assertEqual(report["dropped_overlength"], 1) + + def test_malformed_overlength_still_fails_validation(self): + with self.assertRaisesRegex(ValueError, "mismatched"): + self.load([dict(input_ids=[1] * 65537, loss_mask=[1, 1])]) + with self.assertRaisesRegex(ValueError, "must be 0 or 1"): + self.load([dict(input_ids=[1] * 65537, loss_mask=[1] * 65536 + [2])]) + + def test_raw_conversations_cannot_silently_truncate(self): + with self.assertRaisesRegex(ValueError, "requires pre-tokenized"): + self.load([{"conversations": [{"role": "user", "content": "hello"}]}]) + + def test_schema_requires_native_source_and_defaults_to_legacy(self): + self.assertEqual( + DataConfig(prompts_path="data.jsonl").overlength_policy, "truncate" + ) + self.assertEqual( + DataConfig( + prompts_path="data.jsonl", overlength_policy="drop" + ).overlength_policy, + "drop", + ) + with self.assertRaisesRegex( + ValidationError, "requires native data.prompts_path" + ): + DataConfig(train_data_path="raw.jsonl", overlength_policy="drop") + with self.assertRaises(ValidationError): + DataConfig(prompts_path="data.jsonl", overlength_policy="invalid") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_optimizer/test_torchspec_compat.py b/tests/test_optimizer/test_torchspec_compat.py new file mode 100644 index 0000000000..7919e8b4e4 --- /dev/null +++ b/tests/test_optimizer/test_torchspec_compat.py @@ -0,0 +1,42 @@ +import copy +import math +import unittest +from unittest.mock import patch +import torch +from specforge.config.schema import TrainingConfig +from specforge.optimizer import BF16Optimizer + +class TorchSpecCompatibility(unittest.TestCase): + def make(self): + model=torch.nn.Linear(2,2,bias=False).to(torch.bfloat16) + opt=BF16Optimizer(model,lr=1e-4,weight_decay=.01,min_lr=1e-5,total_steps=1197,warmup_ratio=.01,lr_scheduler='torchspec_cosine') + return model,opt + + def test_original_schedule_and_decay(self): + model,opt=self.make() + self.assertEqual(opt.optimizer.param_groups[0]['weight_decay'],.01) + self.assertEqual(opt.get_learning_rate(),0) + for step in range(1,1198): + model.weight.grad=torch.ones_like(model.weight) + opt.step() + expected=1e-4*step/11 if step<=11 else 1e-5+.5*(1e-4-1e-5)*(1+math.cos(math.pi*(step-11)/(1197-11))) + self.assertAlmostEqual(opt.get_learning_rate(),expected,places=14) + + def test_checkpoint_next_update(self): + model,opt=self.make() + for _ in range(12): + model.weight.grad=torch.ones_like(model.weight);opt.step() + other,restored=self.make() + other.load_state_dict(model.state_dict()) + with patch("specforge.optimizer.print_on_rank0"): + restored.load_state_dict(copy.deepcopy(opt.state_dict())) + model.weight.grad=torch.full_like(model.weight,.25) + other.weight.grad=model.weight.grad.clone() + opt.step();restored.step() + self.assertEqual(opt.get_learning_rate(),restored.get_learning_rate()) + self.assertTrue(torch.equal(model.weight,other.weight)) + for a,b in zip(opt.fp32_params,restored.fp32_params):self.assertTrue(torch.equal(a,b)) + + def test_config_rejects_invalid_min_lr(self): + for kw in [dict(learning_rate=1e-4,min_lr=.1),dict(lr_scheduler='constant',min_lr=1e-5)]: + with self.assertRaises(ValueError):TrainingConfig(**kw) diff --git a/tests/test_runtime/test_capture_begin_parallel.py b/tests/test_runtime/test_capture_begin_parallel.py new file mode 100644 index 0000000000..06953d66e3 --- /dev/null +++ b/tests/test_runtime/test_capture_begin_parallel.py @@ -0,0 +1,202 @@ +"""Begin/recovery singleflight is scoped to original identities, never the store.""" +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from unittest import mock + +from specforge.inference.capture_protocol import CaptureProtocol +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from tests.test_runtime.test_server_capture import _FakeMooncakeStore + + +class Transport: + def __init__(self): + self.calls = [] + self.gate = None + self.lock = threading.Lock() + self.lose_batch = False + + def __call__(self, url, *, json_body, timeout): + body = dict(json_body) + with self.lock: + self.calls.append((url, body)) + if self.gate: + self.gate(url, body) + tasks = body['tasks'] if body['action'] == 'begin_batch' else [body] + replies = [{**task, 'reservation_token': task['attempt_id'] + '-token', + **({'object_key_layout': 'attempt-v1'} if task.get('publication_ownership') == 'attempt-v1' else {})} + for task in tasks] + if body['action'] == 'begin_batch': + if self.lose_batch: + self.lose_batch = False + raise TimeoutError('lost batch reply after server reservation') + return {'attempts': replies} + return replies[0] + + +class ObservedMetadataLock: + def __init__(self, wrapped, observed): + self.wrapped, self.observed = wrapped, observed + + def __enter__(self): + self.wrapped.acquire() + return self + + def __exit__(self, *exc): + self.wrapped.release() + if threading.current_thread().name.startswith('recovery'): + self.observed.set() + + +class BeginParallelTests(unittest.TestCase): + def setUp(self): + self.transport = Transport() + self.store = MooncakeFeatureStore(store=_FakeMooncakeStore(), store_id='store') + self.cap = dict(server_epoch='epoch', begin_batch=True, published_ownership='attempt-v1') + self.first = CaptureProtocol('http://endpoint-1', self.transport, 1) + self.second = CaptureProtocol('http://endpoint-2', self.transport, 1) + + def request(self, sid, attempt=None): + return (dict(store_id='store', sample_id=sid, gen=1, features={'aux': 'hidden'}, passthrough=[]), + [1, 2, 3], attempt or sid + '-attempt', ['hidden']) + + def begin(self, protocol, sid): + return self.store.begin_capture_attempts(protocol, self.cap, [self.request(sid)])[0] + + def test_two_endpoints_enter_http_concurrently(self): + barrier = threading.Barrier(2, timeout=2) + self.transport.gate = lambda url, body: barrier.wait() + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(self.begin, self.first, 'first') + second = executor.submit(self.begin, self.second, 'second') + self.assertEqual(first.result(timeout=3)['attempt_id'], 'first-attempt') + self.assertEqual(second.result(timeout=3)['attempt_id'], 'second-attempt') + self.assertEqual(len(self.transport.calls), 2) + self.assertFalse(self.store._capture_pending_begin) + + def test_fresh_prepare_does_not_wait_for_other_inflight_http(self): + entered, release = threading.Event(), threading.Event() + def gate(url, body): + entered.set() + self.assertTrue(release.wait(2)) + self.transport.gate = gate + with ThreadPoolExecutor(max_workers=2) as executor: + first = executor.submit(self.begin, self.first, 'blocked') + try: + self.assertTrue(entered.wait(1)) + executor.submit(self.store.prepare_capture_retry, 'unrelated').result(timeout=.5) + self.assertFalse(first.done()) + finally: + release.set() + first.result(timeout=2) + + def test_same_sid_cross_endpoint_cannot_replace_pending_or_active_attempt(self): + entered, release = threading.Event(), threading.Event() + def gate(url, body): + entered.set() + self.assertTrue(release.wait(2)) + self.transport.gate = gate + with ThreadPoolExecutor(max_workers=1) as executor: + first = executor.submit(self.begin, self.first, 'same') + try: + self.assertTrue(entered.wait(1)) + original = self.store._capture_pending_begin['same'] + with self.assertRaisesRegex(RuntimeError, 'pending'): + self.begin(self.second, 'same') + self.assertIs(self.store._capture_pending_begin['same'], original) + self.assertEqual(len(self.transport.calls), 1) + finally: + release.set() + first.result(timeout=2) + with self.assertRaisesRegex(RuntimeError, 'prior capture'): + self.begin(self.second, 'same') + self.assertEqual(len(self.transport.calls), 1) + + def test_waiting_recovery_does_not_replay_completed_original_flight(self): + entered, release, snapshot = threading.Event(), threading.Event(), threading.Event() + def gate(url, body): + entered.set() + self.assertTrue(release.wait(2)) + self.transport.gate = gate + self.store._lock = ObservedMetadataLock(self.store._lock, snapshot) + with ThreadPoolExecutor(max_workers=1) as original_executor, ThreadPoolExecutor(max_workers=1, thread_name_prefix='recovery') as recovery_executor: + original = original_executor.submit(self.begin, self.first, 'same') + try: + self.assertTrue(entered.wait(1)) + recovery = recovery_executor.submit(self.store._recover_capture_begin, 'same') + self.assertTrue(snapshot.wait(1)) + finally: + release.set() + original.result(timeout=2) + self.assertIsNone(recovery.result(timeout=2)) + self.assertEqual(len(self.transport.calls), 1) + + def test_waiting_recovery_never_retargets_replacement_intent(self): + self.transport.lose_batch = True + with self.assertRaises(TimeoutError): + self.begin(self.first, 'same') + original = self.store._capture_pending_begin['same'] + snapshot = threading.Event() + self.store._lock = ObservedMetadataLock(self.store._lock, snapshot) + replacement = replace(original, protocol=self.second, capability=dict(self.cap, server_epoch='new-epoch'), + attempt_id='replacement', flight=threading.RLock()) + with original.flight: + with ThreadPoolExecutor(max_workers=1, thread_name_prefix='recovery') as executor: + recovery = executor.submit(self.store._recover_capture_begin, 'same') + self.assertTrue(snapshot.wait(1)) + with self.store._lock: + self.store._capture_pending_begin['same'] = replacement + # The executor must not join while this thread owns its flight. + original.flight.release() + try: + self.assertIsNone(recovery.result(timeout=2)) + finally: + original.flight.acquire() + self.assertIs(self.store._capture_pending_begin['same'], replacement) + self.assertEqual(len(self.transport.calls), 1) + + def test_partial_track_failure_preserves_only_original_unfinished_intent(self): + original_track = self.store.track_external_attempt + failures = [True] + def track(sid, **kwargs): + if sid == 'second' and failures[0]: + failures[0] = False + raise RuntimeError('metadata tracking failed') + return original_track(sid, **kwargs) + with mock.patch.object(self.store, 'track_external_attempt', side_effect=track): + with self.assertRaisesRegex(RuntimeError, 'tracking failed'): + self.store.begin_capture_attempts(self.first, self.cap, [self.request('first'), self.request('second')]) + self.assertNotIn('first', self.store._capture_pending_begin) + intent = self.store._capture_pending_begin['second'] + wire = self.store._recover_capture_begin('second') + self.assertEqual(wire['reservation_token'], 'second-attempt-token') + self.assertEqual(intent.protocol.url, self.first.url) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual([body['action'] for _, body in self.transport.calls], ['begin_batch', 'begin']) + + def test_lost_batch_recovery_retains_original_endpoint_epoch_and_attempts(self): + self.transport.lose_batch = True + with self.assertRaises(TimeoutError): + self.store.begin_capture_attempts(self.first, self.cap, [self.request('first'), self.request('second')]) + intents = dict(self.store._capture_pending_begin) + self.store._recover_capture_begin('second') + self.store._recover_capture_begin('first') + for url, body in self.transport.calls[1:]: + self.assertEqual(url, self.first.url) + self.assertEqual(body['server_epoch'], 'epoch') + self.assertEqual(body['attempt_id'], intents[body['sample_id']].attempt_id) + self.assertFalse(self.store._capture_pending_begin) + + def test_legacy_single_begin_fallback_reenters_batch_flight(self): + self.cap.pop('begin_batch') + with ThreadPoolExecutor(max_workers=1) as executor: + replies = executor.submit(self.store.begin_capture_attempts, self.first, self.cap, + [self.request('first'), self.request('second')]).result(timeout=2) + self.assertEqual(len(replies), 2) + self.assertEqual([body['action'] for _, body in self.transport.calls], ['begin', 'begin']) + self.assertFalse(self.store._capture_pending_begin) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_capture_fencing.py b/tests/test_runtime/test_capture_fencing.py new file mode 100644 index 0000000000..97f047ee8d --- /dev/null +++ b/tests/test_runtime/test_capture_fencing.py @@ -0,0 +1,253 @@ +"""Attempt-fence integration through real adapter and separate FeatureStores.""" +import uuid +import unittest +from unittest import mock + +from specforge.inference.adapters.server_capture import SGLangServerCaptureAdapter as _NativeAdapter +from specforge.inference.capture_protocol import CaptureProtocol + +def SGLangServerCaptureAdapter(*args, **kwargs): + adapter = _NativeAdapter(*args, **kwargs) + adapter._capture_protocol = CaptureProtocol(adapter.base_url, kwargs["control_fn"], adapter.timeout_s) + return adapter +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from tests.test_runtime.test_server_capture import ( + _FakeMooncakeStore, _StubCaptureServer, _capture_schema, _task, _eagle3_contract, + HIDDEN, AUX_LAYERS, _request_specs, +) + + +class ProtocolServer: + def __init__(self, backend): + self.backend = backend + self.stub = _StubCaptureServer(backend) + self.stub._result = lambda result: result + self.active = {} + self.retired = {} + self.drained = True + self.events = [] + self.capability = dict(protocol_version=1, server_epoch='epoch', capture_namespace='namespace', + capture_signature='signature', cache_enabled=True, + schema=[dict(name='aux', width=HIDDEN*len(AUX_LAYERS), dtype='float32', layer_ids=list(AUX_LAYERS)), + dict(name='last_hidden', width=HIDDEN, dtype='float32', layer_ids=[])]) + + def __call__(self, url, json_body, timeout): + if url.endswith('/spec_capture/control'): + body = json_body + action = body['action'] + self.events.append(action) + if action == 'capability': + return self.capability + sid = body['sample_id'] + if action == 'begin': + if sid in self.active: + old = self.active[sid] + if old['attempt_id'] == body['attempt_id'] and old['request_digest'] == body['request_digest']: + return dict(old) + raise RuntimeError('prior attempt unclean') + entry = {k: body[k] for k in ('store_id','sample_id','gen','attempt_id','server_epoch')} + entry.update(reservation_token=uuid.uuid4().hex, state='RESERVED', writer_drained=False, + request_digest=body['request_digest'], cleanup_owner=None, cleanup_token=None, readers={}, released_readers=set()) + self.active[sid] = entry + return dict(entry) + entry = self.active.get(sid) + if entry is None or entry['reservation_token'] != body['reservation_token']: + entry = self.retired.get(body['reservation_token']) + if entry is None or entry['reservation_token'] != body['reservation_token']: + raise RuntimeError('unknown attempt') + if action == 'resolve': + if entry['state'] == 'RETIRED': + return dict(entry) + entry['writer_drained'] = self.drained + if body.get('cancel'): + entry['state'] = 'WRITER_DRAINED' if self.drained else 'DRAINING' + return dict(entry) + if action == 'read_acquire': + if entry['state'] != 'PUBLISHED' or entry['cleanup_token']: + raise RuntimeError('attempt is not readable') + owner = body['reader_owner'] + token = entry['readers'].setdefault(owner, uuid.uuid4().hex) + return dict(entry,reader_owner=owner,reader_token=token) + if action == 'read_release': + owner,token = body['reader_owner'],body['reader_token'] + if entry['readers'].get(owner) == token: + entry['readers'].pop(owner) + entry['released_readers'].add((owner,token)) + elif (owner,token) not in entry['released_readers']: + raise RuntimeError('unknown read permit') + return dict(entry,reader_owner=owner,reader_token=token,reader_released=True) + if action == 'cleanup_acquire': + if entry['readers']: + raise RuntimeError('active readers') + if entry['state'] == 'RETIRED': + return dict(entry) + if not entry['writer_drained']: + raise RuntimeError('writer still live') + if entry['cleanup_owner'] not in (None, body['cleanup_owner']): + raise RuntimeError('cleanup owned by another store') + entry['cleanup_owner'] = body['cleanup_owner'] + entry['cleanup_token'] = entry['cleanup_token'] or uuid.uuid4().hex + return dict(entry) + if action == 'cleanup_done': + if body.get('cleanup_token') != entry['cleanup_token']: + raise RuntimeError('cleanup token mismatch') + if not entry['writer_drained']: + raise RuntimeError('writer still live') + if any(key.startswith(f"{entry['store_id']}/{sid}/g1/") for key in self.backend._d): + raise RuntimeError('cleanup ack before key removal') + entry['state'] = 'RETIRED' + self.retired[entry['reservation_token']] = entry + self.active.pop(sid, None) + return dict(entry) + raise RuntimeError(action) + rows = self.stub(url, json_body, timeout) + for row, spec, tokens in zip(rows, _request_specs(json_body), json_body['input_ids']): + entry = self.active[spec['sample_id']] + entry.update(state='PUBLISHED', writer_drained=True) + row['meta_info']['spec_capture'].update(entry) + row['meta_info']['spec_capture'].update({k:self.capability[k] for k in ('protocol_version','capture_namespace','capture_signature')}) + row['meta_info']['spec_capture'].update(H=0, L=0, N=len(tokens), span=[0,len(tokens)], execution_epoch=0, reused_tokens=0) + return rows + + +class FenceTests(unittest.TestCase): + def setUp(self): + self.backend = _FakeMooncakeStore() + self.server = ProtocolServer(self.backend) + self.producer = MooncakeFeatureStore(store=self.backend, store_id='run0') + self.adapter = SGLangServerCaptureAdapter('http://server', self.producer, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, + timeout_s=0.01) + + def produce(self, index=0): + return self.adapter.produce_refs([_task(index, 5)], capture=_eagle3_contract())[0] + + def test_separate_consumer_release_carries_fence_and_retires_registry(self): + ref = self.produce() + self.assertIn('capture_fence', ref.metadata) + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', self.server): + tensors, handle = consumer.get(ref) + self.assertEqual(tensors['hidden_state'].shape[1], 5) + consumer.release(handle) + self.assertFalse(self.backend._d) + self.assertFalse(self.server.active) + self.assertIn('cleanup_done', self.server.events) + with mock.patch('specforge.inference.capture_protocol.http_control', self.server): + with self.assertRaises(KeyError): + consumer.get(ref) + + def test_timeout_retains_keys_until_writer_drain(self): + payload = self.adapter._spec_capture_payload(_task(0,5)) + wire = self.adapter._capture_protocol.begin(self.server.capability, payload, [1]*5, 'first') + self.producer.track_capture_fence(wire, ['hidden_state'], control_fn=self.server) + self.producer.track_external_attempt(payload['sample_id'], generation=1, feature_names=['hidden_state']) + self.backend._d[f"run0/{payload['sample_id']}/g1/hidden_state"] = b'old' + self.server.drained = False + with self.assertRaises(TimeoutError): + self.producer.prepare_capture_retry(payload['sample_id']) + self.assertTrue(self.backend._d) + self.assertTrue(self.producer._external_provisional) + self.server.drained = True + self.producer.prepare_capture_retry(payload['sample_id']) + self.assertFalse(self.backend._d) + self.assertFalse(self.producer._capture_fences) + + def test_old_cleanup_and_duplicate_release_cannot_delete_new_g1(self): + ref = self.produce() + old_token = ref.metadata['capture_fence']['reservation_token'] + values, handle = self.producer.get(ref) + self.producer.release(handle) + new_ref = self.produce() + new_token = new_ref.metadata['capture_fence']['reservation_token'] + self.assertNotEqual(old_token,new_token) + before = dict(self.backend._d) + self.producer.abort(ref.sample_id, attempt_token=old_token) + self.producer.release(handle) + self.assertEqual(before,self.backend._d) + with self.assertRaises(KeyError): + self.producer.get(ref) + values, new_handle = self.producer.get(new_ref) + self.producer.release(new_handle) + self.assertFalse(self.backend._d) + + def test_old_producer_gc_cannot_remove_new_cross_process_g1(self): + ref = self.produce() + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', self.server): + _, handle = consumer.get(ref) + consumer.release(handle) + replacement_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + replacement_adapter = SGLangServerCaptureAdapter('http://server', replacement_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server) + replacement_adapter.produce_refs([_task(0,5)], capture=_eagle3_contract()) + before = dict(self.backend._d) + self.producer.max_hold_age_s = 0 + self.producer.gc(now=10**12) + self.assertEqual(before, self.backend._d) + self.assertFalse(self.producer._capture_fences) + + def test_lost_begin_response_recovers_same_intent_and_cleans(self): + original = self.adapter._capture_protocol.post + lost = [False] + def transport(url,json_body,timeout): + reply = original(url,json_body,timeout) + if json_body.get('action') == 'begin' and not lost[0]: + lost[0] = True + raise TimeoutError('begin response lost after reservation') + return reply + self.adapter._capture_protocol.post = transport + with self.assertRaises(TimeoutError): + self.produce() + self.assertTrue(self.producer._capture_pending_begin) + self.assertTrue(self.server.active) + self.producer.discard_external_attempts() + self.assertFalse(self.producer._capture_pending_begin) + self.assertFalse(self.server.active) + self.assertFalse(self.producer._capture_fences) + + def test_lost_read_acquire_reply_recovers_intent_before_cleanup(self): + ref=self.produce() + fence=self.producer._capture_fences[ref.sample_id] + original=fence.post + lost=[True] + def transport(url,json_body,timeout): + reply=original(url,json_body,timeout) + if json_body.get('action')=='read_acquire' and lost[0]: + lost[0]=False + raise TimeoutError('read acquire reply lost') + return reply + fence.post=transport + with self.assertRaises(TimeoutError): + self.producer.get(ref) + self.assertEqual(len(self.server.active[ref.sample_id]['readers']),1) + self.assertEqual(self.producer.health()['capture_read_intents'],1) + self.producer.drain_pending_removals(max_attempts=2,retry_interval_s=0) + self.assertEqual(len(self.server.active[ref.sample_id]['readers']),0) + self.assertEqual(self.producer.health()['capture_read_intents'],0) + _,handle=self.producer.get(ref) + self.producer.release(handle) + + def test_live_lease_prevents_same_name_retry(self): + ref = self.produce() + _, handle = self.producer.get(ref) + with self.assertRaisesRegex(RuntimeError,'active consumer'): + self.producer.prepare_capture_retry(ref.sample_id) + self.producer.release(handle) + + def test_negotiated_identity_mismatch_fails_without_adopting(self): + original = self.adapter.post_fn + def corrupt(url,json_body,timeout): + result = original(url,json_body,timeout) + result[0]['meta_info']['spec_capture']['server_epoch'] = 'wrong' + return result + self.adapter.post_fn = corrupt + with self.assertRaisesRegex(RuntimeError,'identity mismatch'): + self.produce() + self.assertTrue(self.producer._external_provisional) + self.producer.discard_external_attempts() + self.assertFalse(self.backend._d) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_capture_producer_coalescing.py b/tests/test_runtime/test_capture_producer_coalescing.py new file mode 100644 index 0000000000..9c660b1dfd --- /dev/null +++ b/tests/test_runtime/test_capture_producer_coalescing.py @@ -0,0 +1,513 @@ +"""Negotiated lease-one producer batching and independent completion contracts.""" +import copy +import os +import tempfile +import threading +import time +import unittest +from concurrent.futures import Future, ThreadPoolExecutor +from unittest import mock + +from specforge.inference.capture_protocol import CaptureProtocol +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from tests.test_runtime.test_capture_fencing import ProtocolServer +from tests.test_runtime.test_disagg_multiserver import _adapter, _build, _prompts +from tests.test_runtime.test_server_capture import _FakeMooncakeStore, _request_specs + + +class BatchServer(ProtocolServer): + def __init__(self, backend): + super().__init__(backend) + self.capability['begin_batch'] = True + for item in self.capability['schema']: + item['dtype'] = 'bfloat16' + self.batches = [] + self.generates = [] + self.gate = None + self.lock = threading.RLock() + + def __call__(self, url, json_body, timeout): + body = copy.deepcopy(json_body) + action = body.get('action', 'generate') + if action == 'begin_batch': + with self.lock: + self.batches.append(body) + replies = [super(BatchServer, self).__call__(url, dict(task, action='begin'), timeout) + for task in body['tasks']] + if self.gate: + self.gate(action, body) + return {'attempts': replies} + if action == 'generate': + with self.lock: + self.generates.append(body) + if self.gate: + self.gate(action, body) + with self.lock: + return super().__call__(url, body, timeout) + + +def wait_for(predicate, timeout=3): + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + raise AssertionError('condition did not become true') + time.sleep(.001) + + +class CoalescedProducerTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.backend = _FakeMooncakeStore() + self.store = MooncakeFeatureStore(store=self.backend, store_id='run0') + self.server = BatchServer(self.backend) + self.channel = StreamingRefChannel(os.path.join(self.directory.name, 'refs')) + + def build(self, n=8, concurrency=4, servers=None, **kwargs): + adapters = [] + for i, server in enumerate(servers or [self.server]): + adapter = _adapter(self.store, server, url=f'http://server{i}') + adapter._capture_protocol = CaptureProtocol(adapter.base_url, server, 1) + adapters.append(adapter) + lease = kwargs.pop('lease', 1) + return _build(adapters, _prompts(n), self.store, self.channel, + lease=lease, producer_concurrency=concurrency, **kwargs) + + def drive_async(self, drive, **kwargs): + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(drive, **kwargs) + self.addCleanup(pool.shutdown) + return future + + def test_initial_idle_slots_share_begin_but_generate_independently(self): + workers, drive = self.build(n=4) + self.assertEqual(drive(), 4) + self.assertEqual([len(b['tasks']) for b in self.server.batches], [4]) + self.assertEqual([len(b['input_ids']) for b in self.server.generates], [1]*4) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_concurrency_twelve_initial_burst_is_one_registration(self): + workers, drive = self.build(n=12, concurrency=12) + self.assertEqual(drive(), 12) + self.assertEqual([len(b['tasks']) for b in self.server.batches], [12]) + self.assertEqual([len(b['input_ids']) for b in self.server.generates], [1]*12) + self.assertEqual(workers[0].health()['in_flight'], 0) + + def test_returned_output_cannot_silently_exceed_reserved_schema(self): + for item in self.server.capability['schema']: + item['dtype'] = 'int8' # Deliberately false server capability. + workers, drive = self.build(n=2, concurrency=2, max_worker_failures=1) + with self.assertRaisesRegex(RuntimeError, 'dead workers'): + drive() + self.assertIn('exceeded negotiated byte reservation', self.channel.failure()) + self.assertEqual(self.channel.published, 0) + self.assertFalse(self.backend._d) + self.assertFalse(self.server.active) + self.assertEqual(workers[0].health()['in_flight'], 0) + + def test_done_generate_publishes_while_followup_begin_is_blocked(self): + release_slow, begin_blocked, release_begin = (threading.Event() for _ in range(3)) + slow = {} + def gate(action, body): + if action == 'begin_batch' and len(self.server.batches) == 2: + begin_blocked.set() + self.assertTrue(release_begin.wait(3)) + if action == 'generate': + sid = _request_specs(body)[0]['sample_id'] + if sid == slow['sid']: + self.assertTrue(release_slow.wait(3)) + original = self.server.__call__ + # Capture the initial batch identity before any generate can execute. + def transport(url, json_body, timeout): + if json_body.get('action') == 'begin_batch' and not slow: + slow['sid'] = json_body['tasks'][0]['sample_id'] + return original(url, json_body, timeout) + self.server.gate = gate + workers, drive = self.build(n=7, concurrency=3, servers=[transport]) + future = self.drive_async(drive) + try: + self.assertTrue(begin_blocked.wait(2)) + release_slow.set() + wait_for(lambda: self.channel.published >= 3) + self.assertFalse(future.done()) + self.assertFalse(release_begin.is_set()) + finally: + release_slow.set() + release_begin.set() + self.assertEqual(future.result(timeout=3), 7) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertGreater(len(self.server.batches), 1) + + def test_late_idle_slots_coalesce_after_inflight_begin(self): + second_begin = threading.Event() + release_begin = threading.Event() + release_initial = threading.Event() + initial = [] + original = self.server.__call__ + def transport(url, json_body, timeout): + if json_body.get('action') == 'begin_batch' and not initial: + initial.extend(task['sample_id'] for task in json_body['tasks']) + return original(url, json_body, timeout) + def gate(action, body): + if action == 'begin_batch' and len(self.server.batches) == 2: + second_begin.set() + self.assertTrue(release_begin.wait(3)) + if action == 'generate': + sid = _request_specs(body)[0]['sample_id'] + if sid in initial[1:]: + self.assertTrue(release_initial.wait(3)) + self.server.gate = gate + workers, drive = self.build(n=9, concurrency=4, servers=[transport]) + source = workers[0].feature_source + prepared_late = set() + prepare = source.prepare_refs_batches + def observe_prepare(batches, *, capture): + result = prepare(batches, capture=capture) + if second_begin.is_set(): + prepared_late.update(task.task_id for tasks in batches for task in tasks) + return result + source.prepare_refs_batches = observe_prepare + future = self.drive_async(drive) + try: + self.assertTrue(second_begin.wait(2)) + release_initial.set() + wait_for(lambda: len(prepared_late) == 3) + finally: + release_initial.set() + release_begin.set() + self.assertEqual(future.result(timeout=3), 9) + self.assertEqual([len(b['tasks']) for b in self.server.batches[:3]], [4, 1, 3]) + + def test_other_endpoint_registers_while_first_endpoint_is_blocked(self): + entered, other_entered, release = (threading.Event() for _ in range(3)) + other = BatchServer(self.backend) + def first_gate(action, body): + if action == 'begin_batch': + entered.set() + self.assertTrue(release.wait(3)) + def other_gate(action, body): + if action == 'begin_batch': + self.assertTrue(entered.wait(2)) + other_entered.set() + self.server.gate, other.gate = first_gate, other_gate + _, drive = self.build(n=8, concurrency=2, servers=[self.server, other]) + future = self.drive_async(drive) + try: + self.assertTrue(other_entered.wait(2)) + finally: + release.set() + self.assertEqual(future.result(timeout=3), 8) + + def test_generate_submit_failure_retires_registered_preleases(self): + workers, drive = self.build(n=4) + submit = ThreadPoolExecutor.submit + def guarded_submit(executor, fn, *args, **kwargs): + if getattr(fn, '__name__', '') == 'run_leased': + raise RuntimeError('injected generate submit failure') + return submit(executor, fn, *args, **kwargs) + with mock.patch.object(ThreadPoolExecutor, 'submit', guarded_submit): + with self.assertRaisesRegex(RuntimeError, 'generate submit failure'): + drive() + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + self.assertIsNotNone(self.channel.failure()) + + def test_cancelled_generate_future_releases_prelease_and_fence(self): + workers, drive = self.build(n=4, max_worker_failures=1) + submit = ThreadPoolExecutor.submit + def cancelling_submit(executor, fn, *args, **kwargs): + if getattr(fn, '__name__', '') == 'run_leased': + future = Future() + future.cancel() + return future + return submit(executor, fn, *args, **kwargs) + with mock.patch.object(ThreadPoolExecutor, 'submit', cancelling_submit): + with self.assertRaisesRegex(RuntimeError, 'dead workers'): + drive() + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_stale_prepared_cleanup_preserves_peer_replacement_attempt(self): + workers, _drive = self.build(n=1, concurrency=2) + source = workers[0].feature_source + # Use real task leases and store fences. + workers[0].start() + workers[0].controller.ingest_prompts(_prompts(1)) + tasks = workers[0].lease_tasks(1) + old = source.prepare_refs_batches([tasks], capture=workers[0].capture) + source.register_prepared_batches(old) + sid = old[0]['payloads'][0]['sample_id'] + self.store.prepare_capture_retry(sid) # A peer retires the old writer. + new = source.prepare_refs_batches([tasks], capture=workers[0].capture) + source.register_prepared_batches(new) + fence = self.store._capture_fences[sid] + source.discard_prepared_batches(old) + self.assertIs(self.store._capture_fences[sid], fence) + self.assertEqual(self.server.active[sid]['reservation_token'], fence.token) + source.discard_prepared_batches(new) + workers[0].fail_leased(tasks, 'test-cleanup', unhealthy=False) + self.assertFalse(self.server.active) + + def test_unknown_old_cleanup_never_recovers_replacement_pending_intent(self): + workers, _drive = self.build(n=1, concurrency=2) + source = workers[0].feature_source + workers[0].start() + workers[0].controller.ingest_prompts(_prompts(1)) + tasks = workers[0].lease_tasks(1) + calls = [] + original = source._capture_protocol.post + def lose_batch(url, json_body, timeout): + calls.append(copy.deepcopy(json_body)) + reply = original(url, json_body, timeout) + if json_body.get('action') == 'begin_batch': + raise TimeoutError('lost begin response') + return reply + source._capture_protocol.post = lose_batch + old = source.prepare_refs_batches([tasks], capture=workers[0].capture) + with self.assertRaises(TimeoutError): + source.register_prepared_batches(old) + sid = old[0]['payloads'][0]['sample_id'] + self.store.prepare_capture_retry(sid) + new = source.prepare_refs_batches([tasks], capture=workers[0].capture) + with self.assertRaises(TimeoutError): + source.register_prepared_batches(new) + pending = self.store._capture_pending_begin[sid] + calls_before = len(calls) + source.discard_prepared_batches(old) + self.assertEqual(len(calls), calls_before) + self.assertIs(self.store._capture_pending_begin[sid], pending) + self.assertEqual(self.server.active[sid]['attempt_id'], pending.attempt_id) + source.discard_prepared_batches(new) + workers[0].fail_leased(tasks, 'test-cleanup', unhealthy=False) + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + + def test_unknown_begin_reply_recovers_original_intents_before_retry(self): + calls = [] + original = self.server.__call__ + lost = [False] + def transport(url, json_body, timeout): + calls.append(copy.deepcopy(json_body)) + reply = original(url, json_body, timeout) + if json_body.get('action') == 'begin_batch' and not lost[0]: + lost[0] = True + raise TimeoutError('lost begin reply') + return reply + workers, drive = self.build(n=4, servers=[transport]) + self.assertEqual(drive(), 4) + first = next(body for body in calls if body.get('action') == 'begin_batch') + recovered = [body for body in calls if body.get('action') == 'begin'] + self.assertEqual({b['attempt_id'] for b in recovered}, {b['attempt_id'] for b in first['tasks']}) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + + def test_http_409_refresh_never_retargets_original_begin_intents(self): + import requests + calls = [] + original = self.server.__call__ + rejected = [False] + recovered = [] + def transport(url, json_body, timeout): + calls.append(copy.deepcopy(json_body)) + if json_body.get('action') == 'begin_batch' and not rejected[0]: + # A response encoding failure may surface as HTTP 409 after + # reservations were granted. It is not proof of rollback. + original(url, json_body, timeout) + rejected[0] = True + self.server.capability['server_epoch'] = 'refreshed-epoch' + response = requests.Response() + response.status_code = 409 + raise requests.HTTPError('ambiguous server rejection', response=response) + if json_body.get('action') == 'begin': + recovered.append(copy.deepcopy(json_body)) + # Cleanup must recover the exact old reservation despite the + # refreshed discovery; reject any new identity immediately. + self.assertEqual(json_body['server_epoch'], 'epoch') + return original(url, json_body, timeout) + workers, drive = self.build(n=4, servers=[transport], max_worker_failures=1) + with self.assertRaisesRegex(RuntimeError, 'dead workers'): + drive() + batch = next(body for body in calls if body.get('action') == 'begin_batch') + self.assertEqual({body['attempt_id'] for body in recovered}, + {body['attempt_id'] for body in batch['tasks']}) + self.assertEqual(len([body for body in calls if body.get('action') == 'begin_batch']), 1) + self.assertFalse(self.server.generates) + self.assertFalse(self.store._capture_pending_begin) + self.assertFalse(self.server.active) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_unreturned_outputs_consume_global_hard_cap_credit(self): + # Both negotiated bfloat16 engine tensors plus two int64 passthrough tensors. + per_token = sum(item['width'] * 2 for item in self.server.capability['schema']) + 16 + cap = per_token * 42 # Fits two longest prompts across both endpoints. + other = BatchServer(self.backend) + _, drive = self.build(n=16, concurrency=8, servers=[self.server, other], + feature_store_max_resident_bytes=cap) + reader = StreamingRefChannel(self.channel.path) + outstanding = 0 + peak = 0 + lock = threading.Lock() + def gate(action, body): + nonlocal outstanding, peak + if action == 'begin_batch': + # Derive output lengths from request ids in this fixed fixture. + size = sum((6 + int(task['sample_id'].split(':p')[1])) * per_token for task in body['tasks']) + with lock: + outstanding += size + peak = max(peak, outstanding) + self.assertLessEqual(peak, cap) + self.server.gate = other.gate = gate + future = self.drive_async(drive) + while not future.done(): + refs = reader.poll() + if refs: + # Acknowledge the actual completed outputs. Not-yet-returned + # writers remain in outstanding and continue occupying credit. + for ref in refs: + self.store.abort(ref.sample_id, reason='test-consumer-completed') + with lock: + outstanding -= sum(ref.estimated_bytes for ref in refs) + reader.mark_consumed(len(refs)) + time.sleep(.001) + self.assertEqual(future.result(timeout=3), 16) + self.assertLessEqual(peak, cap) + + def test_per_task_errors_retire_every_registered_attempt(self): + self.server.stub.error_sample_ids = {f'run0:p{i}' for i in range(4)} + workers, drive = self.build(n=4, max_prompt_attempts=1, + feature_store_max_resident_bytes=100000) + with self.assertRaisesRegex(RuntimeError, 'terminally failed prompt'): + drive() + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_fences) + self.assertFalse(self.store._capture_pending_begin) + self.assertFalse(self.backend._d) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_partial_lease_errors_cleanup_only_unreturned_identities(self): + self.server.stub.error_sample_ids = {'run0:p1', 'run0:p3'} + workers, drive = self.build(n=4, concurrency=2, lease=2, max_prompt_attempts=1, + feature_store_max_resident_bytes=100000) + with self.assertRaisesRegex(RuntimeError, 'terminally failed prompt'): + drive() + refs = self.channel.poll() + self.assertEqual({ref.sample_id for ref in refs}, {'run0:p0', 'run0:p2'}) + self.assertEqual(set(self.server.active), {'run0:p0', 'run0:p2'}) + self.assertTrue(self.backend._d) + self.assertEqual(set(self.store._capture_fences), {'run0:p0', 'run0:p2'}) + # This fixture negotiates legacy ownership, so successful fences stay + # live until consumer release; failed fences alone must be retired. + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def _stop_active_worker(self, method): + entered, release = threading.Event(), threading.Event() + calls = 0 + lock = threading.Lock() + def gate(action, body): + nonlocal calls + if action == 'generate': + with lock: + calls += 1 + if calls == 4: + entered.set() + self.assertTrue(release.wait(3)) + self.server.gate = gate + workers, drive = self.build(n=8) + future = self.drive_async(drive) + try: + self.assertTrue(entered.wait(2)) + getattr(workers[0], method)() + finally: + release.set() + with self.assertRaisesRegex(RuntimeError, 'stopped or draining with prompts remaining'): + future.result(timeout=3) + self.assertEqual(self.channel.published, 4) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + self.assertEqual(workers[0].controller.status()['prompts_pending'], 4) + + def test_stopped_worker_leaves_coordinator_without_spinning(self): + self._stop_active_worker('stop') + + def test_draining_worker_leaves_coordinator_without_spinning(self): + self._stop_active_worker('drain') + + def test_stopped_endpoint_releases_pending_plan_to_surviving_worker(self): + entered, release = threading.Event(), threading.Event() + other = BatchServer(self.backend) + def first_gate(action, body): + if action == 'generate': + entered.set() + self.assertTrue(release.wait(3)) + def other_gate(action, body): + if action == 'generate': + self.assertTrue(release.wait(3)) + self.server.gate, other.gate = first_gate, other_gate + workers, drive = self.build(n=12, concurrency=2, servers=[self.server, other]) + future = self.drive_async(drive) + try: + self.assertTrue(entered.wait(2)) + workers[0].stop() + finally: + release.set() + self.assertEqual(future.result(timeout=3), 12) + self.assertEqual(workers[0].health()['state'], 'stopped') + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_stop_after_registration_cleans_without_submitting_generate(self): + stop = threading.Event() + workers, drive = self.build(n=4, feature_store_max_resident_bytes=100000) + source = workers[0].feature_source + register = source.register_prepared_batches + def stopping_register(prepared): + register(prepared) + stop.set() + source.register_prepared_batches = stopping_register + self.assertEqual(drive(should_stop=stop.is_set), 0) + self.assertEqual([len(b['tasks']) for b in self.server.batches], [4]) + self.assertFalse(self.server.generates) + self.assertFalse(self.server.active) + self.assertFalse(self.store._capture_pending_begin) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + self.assertTrue(self.channel.is_closed()) + + def test_partial_preparation_is_loud_and_releases_every_lease(self): + workers, drive = self.build(n=4, max_worker_failures=1) + source = workers[0].feature_source + prepare = source.prepare_refs_batches + def short_prepare(groups, *, capture): + return prepare(groups, capture=capture)[:-1] + source.prepare_refs_batches = short_prepare + with self.assertRaisesRegex(RuntimeError, 'preparation returned 3 groups for 4 leases'): + drive() + self.assertFalse(self.server.batches) + self.assertFalse(self.server.generates) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + def test_stop_releases_prepared_leases_waiting_for_credit(self): + per_token = sum(item['width'] * 2 for item in self.server.capability['schema']) + 16 + stop = threading.Event() + workers, drive = self.build(n=6, feature_store_max_resident_bytes=per_token*11) + future = self.drive_async(drive, should_stop=stop.is_set) + wait_for(lambda: self.channel.published >= 1) + stop.set() + self.assertGreaterEqual(future.result(timeout=3), 1) + self.assertFalse(self.server.active.keys() - {ref.sample_id for ref in self.channel.poll()}) + self.assertEqual(workers[0].health()['in_flight'], 0) + self.assertEqual(workers[0].controller.status()['prompts_leased'], 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_capture_transfer.py b/tests/test_runtime/test_capture_transfer.py new file mode 100644 index 0000000000..cb98be100a --- /dev/null +++ b/tests/test_runtime/test_capture_transfer.py @@ -0,0 +1,258 @@ +"""Immutable published ownership, independent producer/consumer lifecycles.""" +import types +import unittest +from unittest import mock + +from specforge.inference.capture_protocol import CaptureProtocol, WriterFence, attempt_keys +from specforge.inference.adapters.server_capture import SGLangServerCaptureAdapter +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from tests.test_runtime.test_capture_fencing import ProtocolServer +from tests.test_runtime.test_server_capture import _FakeMooncakeStore, _task, _capture_schema, _eagle3_contract + + +class TransferServer(ProtocolServer): + def __init__(self, backend): + super().__init__(backend) + self.capability.update(begin_batch=True, published_ownership='attempt-v1') + for feature in self.capability['schema']: + feature['dtype'] = 'bfloat16' + self.calls = [] + + def __call__(self, url, json_body, timeout): + action = json_body.get('action', 'generate') + self.calls.append(action) + if action == 'begin_batch': + replies = [] + for body in json_body['tasks']: + reply = super().__call__(url, dict(body, action='begin'), timeout) + reply['object_key_layout'] = 'attempt-v1' + replies.append(reply) + return dict(attempts=replies) + if action == 'begin': + reply = super().__call__(url, json_body, timeout) + reply['object_key_layout'] = 'attempt-v1' + return reply + rows = super().__call__(url, json_body, timeout) + if action != 'generate': + return rows + for row in rows: + result = row['meta_info']['spec_capture'] + keys = attempt_keys(result, result['features']) + for name, key in keys.items(): + self.backend._d[key] = self.backend._d.pop(f"{result['store_id']}/{result['sample_id']}/g1/{name}") + result['features'][name]['key'] = key + result.update(object_key_layout='attempt-v1', publication_ownership='attempt-v1', physical_keys=keys) + entry = self.active.pop(result['sample_id']) + certificate = {k: result[k] for k in ('store_id', 'sample_id', 'gen', 'attempt_id', 'server_epoch', + 'reservation_token', 'object_key_layout', 'publication_ownership', 'physical_keys', 'writer_drained', 'state')} + entry.update(state='RETIRED', publication_certificate=certificate, + publication_ownership='attempt-v1', physical_keys=keys) + self.retired[entry['reservation_token']] = entry + return rows + + +class TransferTests(unittest.TestCase): + def setUp(self): + self.backend = _FakeMooncakeStore() + self.server = TransferServer(self.backend) + self.producer = MooncakeFeatureStore(store=self.backend, store_id='run0', max_hold_age_s=0) + self.adapter = SGLangServerCaptureAdapter('http://server', self.producer, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + + def produce(self, *indexes): + return self.adapter.produce_refs([_task(i, 5) for i in indexes], capture=_eagle3_contract()) + + def test_batched_begin_cached_capability_and_zero_consumer_rpc(self): + refs = self.produce(0, 1) + self.produce(2) + self.assertEqual(self.server.calls.count('capability'), 1) + self.assertEqual(self.server.calls.count('begin_batch'), 2) + self.assertNotIn('resolve', self.server.calls) + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + before = list(self.server.calls) + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('consumer control RPC')): + for ref in refs: + values, handle = consumer.get(ref) + self.assertEqual(values['hidden_state'].shape[1], 5) + consumer.release(handle) + consumer.drain_pending_removals(retry_interval_s=0) + self.assertEqual(before, self.server.calls) + + def test_multiple_readers_defer_local_remove(self): + ref = self.produce(0)[0] + _, first = self.producer.get(ref) + _, second = self.producer.get(ref) + self.producer.release(first) + self.assertTrue(self.backend._d) + self.assertEqual(len(self.producer._active_leases), 1) + self.producer.release(second) + self.assertFalse(self.backend._d) + + def test_producer_gc_handoff_and_stale_process_cannot_remove_replacement(self): + old = self.produce(0)[0] + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('consumer control RPC')): + _, handle = consumer.get(old) + self.producer.gc(now=10**12) + self.assertTrue(self.backend._d) + self.producer.relinquish_published(old) + self.assertFalse(self.producer._generation) + self.assertTrue(self.backend._d) + replacement_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + replacement = SGLangServerCaptureAdapter('http://server', replacement_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + new = replacement.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + self.assertNotEqual(old.feature_keys, new.feature_keys) + consumer.release(handle) # Only the old attempt's immutable keys. + self.assertEqual(set(self.backend._d), set(new.feature_keys.values())) + self.producer.abort(old.sample_id) + self.producer.gc(now=10**12) + self.assertEqual(set(self.backend._d), set(new.feature_keys.values())) + with self.assertRaises(KeyError): + self.producer.get(old) + + def test_lost_generate_response_resolves_certificate_then_cleans(self): + original = self.adapter.post_fn + def lose(*args, **kwargs): + original(*args, **kwargs) + raise TimeoutError('lost after successful publication') + self.adapter.post_fn = lose + with self.assertRaises(TimeoutError): + self.produce(0) + self.assertTrue(self.backend._d) + self.producer.discard_external_attempts() + self.producer.drain_pending_removals(retry_interval_s=0) + self.assertFalse(self.backend._d) + self.assertFalse(self.producer._capture_fences) + self.assertIn('resolve', self.server.calls) + self.assertNotIn('cleanup_acquire', self.server.calls) + + def test_strict_state_and_exact_keys_fail_closed(self): + ref = self.produce(0)[0] + fence = WriterFence(dict(ref.metadata['capture_fence']), tuple(ref.feature_keys)) + for bad in (dict(fence.wire, state='DRAINING'), dict(fence.wire, writer_drained=False), + dict(fence.wire, physical_keys={})): + with self.assertRaises(RuntimeError): + fence.accept_transfer(bad) + self.assertFalse(fence.transferred) + + def test_lost_begin_batch_reply_recovers_per_task_intents(self): + original = self.adapter._capture_protocol.post + once = [True] + def lose(url, json_body, timeout): + result = original(url, json_body, timeout) + if json_body.get('action') == 'begin_batch' and once[0]: + once[0] = False + raise TimeoutError('lost batched reservations') + return result + self.adapter._capture_protocol.post = lose + with self.assertRaises(TimeoutError): + self.produce(0, 1) + self.assertEqual(len(self.producer._capture_pending_begin), 2) + self.producer.discard_external_attempts() + self.assertFalse(self.server.active) + self.assertFalse(self.producer._capture_pending_begin) + + +class CapabilityTests(unittest.TestCase): + def test_absent_legacy_capability_is_cached(self): + calls = [] + def post(*args, **kwargs): + calls.append(1) + error = RuntimeError('404') + error.response = types.SimpleNamespace(status_code=404) + raise error + protocol = CaptureProtocol('http://server', post, 1) + self.assertIsNone(protocol.capability()) + self.assertIsNone(protocol.capability()) + self.assertEqual(len(calls), 1) + + def test_409_refreshes_capability_without_replaying_mutation(self): + calls = [] + def post(url, json_body, timeout): + calls.append(json_body['action']) + if json_body['action'] == 'capability': + return dict(protocol_version=1, server_epoch=str(len(calls)), capture_namespace='n', + capture_signature='s', schema=[], cache_enabled=False) + error = RuntimeError('409') + error.response = types.SimpleNamespace(status_code=409) + raise error + protocol = CaptureProtocol('http://server', post, 1) + protocol.capability() + with self.assertRaises(RuntimeError): + protocol.call(dict(action='begin')) + self.assertEqual(calls, ['capability', 'begin', 'capability']) + + +if __name__ == '__main__': + unittest.main() + +class PublicationBoundaryTests(TransferTests): + def test_visible_fsync_failure_relinquishes_prefix_and_aborts_suffix(self): + import tempfile + from specforge.launch import _publish_refs_with_cleanup + from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel + refs = self.produce(0, 1) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(directory + '/refs.jsonl') + with mock.patch('specforge.runtime.data_plane.streaming_ref_channel.os.fsync', side_effect=OSError('fsync failed')): + with self.assertRaises(OSError): + _publish_refs_with_cleanup(channel=channel, feature_store=self.producer, refs=refs) + self.assertEqual(channel.published, 1) + self.assertEqual([ref.sample_id for ref in channel.poll()], [refs[0].sample_id]) + self.assertEqual(set(self.backend._d), set(refs[0].feature_keys.values())) + self.assertNotIn(refs[0].sample_id, self.producer._capture_fences) + self.assertNotIn(refs[1].sample_id, self.producer._capture_fences) + + def test_same_consumer_accepts_new_attempt_and_rejects_old_ref(self): + old = self.produce(0)[0] + consumer = MooncakeFeatureStore(store=self.backend, store_id='run0') + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('consumer control RPC')): + _, old_handle = consumer.get(old) + consumer.release(old_handle) + other_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + adapter = SGLangServerCaptureAdapter('http://server', other_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + new = adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + _, new_handle = consumer.get(new) + with self.assertRaises(KeyError): + consumer.get(old) + consumer.release(old_handle) + self.assertTrue(self.backend._d) + consumer.release(new_handle) + self.assertFalse(self.backend._d) + + def test_lost_generate_old_cleanup_cannot_touch_started_new_attempt(self): + original = self.adapter.post_fn + def lose(*args, **kwargs): + original(*args, **kwargs) + raise TimeoutError('lost old publication') + self.adapter.post_fn = lose + with self.assertRaises(TimeoutError): + self.produce(0) + other_store = MooncakeFeatureStore(store=self.backend, store_id='run0') + adapter = SGLangServerCaptureAdapter('http://server', other_store, run_id='run0', + algorithm='eagle3', schema=_capture_schema('eagle3'), post_fn=self.server, control_fn=self.server, hs_capture=True, hs_prefix_cache=True) + new = adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + self.producer.discard_external_attempts() + self.assertEqual(set(self.backend._d), set(new.feature_keys.values())) + +class DelayedAbortTests(TransferTests): + def test_ref_specific_old_abort_does_not_remove_same_store_replacement(self): + from specforge.runtime.data_plane.feature_store import abort_feature_ref + old = self.produce(0)[0] + self.producer.relinquish_published(old) + new = self.produce(0)[0] + before = set(self.backend._d) + abort_feature_ref(self.producer, old, reason='late old publication callback') + self.assertEqual(set(self.backend._d), before) + self.assertTrue(set(new.feature_keys.values()).issubset(self.backend._d)) + + def test_legacy_ref_cannot_abort_same_id_new_capture(self): + from dataclasses import replace + from specforge.runtime.data_plane.feature_store import abort_feature_ref + new = self.produce(0)[0] + legacy = replace(new, metadata={k: v for k, v in new.metadata.items() if k != 'capture_fence'}) + before = set(self.backend._d) + abort_feature_ref(self.producer, legacy, reason='late legacy callback') + self.assertEqual(set(self.backend._d), before) diff --git a/tests/test_runtime/test_hs_controller_parallel_cuda.py b/tests/test_runtime/test_hs_controller_parallel_cuda.py new file mode 100644 index 0000000000..9fc5bac444 --- /dev/null +++ b/tests/test_runtime/test_hs_controller_parallel_cuda.py @@ -0,0 +1,138 @@ +"""Explicitly scheduled real-CUDA checks for concurrent capture landing slots. + +Run on one allocated GPU with SGLANG_RUN_SPEC_FEATURE_CUDA_TESTS=1. The event +wrapper gates only the host's completion observation; all CUDA streams, DMA +copies, and underlying events remain real. CPU discovery never initializes CUDA. +""" + +import json +import os +import threading +import unittest +from unittest import mock + +import torch + +from specforge_sglang_capture.hs import ( + CaptureCancelledError, + CaptureController, + CapturedSpan, + ExecutionIdentity, + FeatureSchema, + FeatureSpec, + PrefixFeatureCache, +) + + +@unittest.skipUnless( + os.environ.get("SGLANG_RUN_SPEC_FEATURE_CUDA_TESTS") == "1", + "requires an explicitly allocated CUDA GPU", +) +class ConcurrentCaptureCudaTests(unittest.TestCase): + def run_pair(self, cancel_first): + self.assertTrue(torch.cuda.is_available()) + schema = FeatureSchema((FeatureSpec("aux", 16, "bfloat16", (1,)), + FeatureSpec("last_hidden", 16, "bfloat16", ()))) + rows = 4096 + sample_bytes = rows * schema.row_bytes + cache = PrefixFeatureCache(schema, 256, sample_bytes * 4) + controller = CaptureController(cache, schema, sample_bytes * 2, + sample_bytes * 4, sample_bytes * 2, max_workers=2) + plans, spans, expected, samples = [], [], [], [] + tokens = tuple(range(rows)) + for i in range(2): + execution = ExecutionIdentity("cuda-probe", str(i), str(i), 0) + namespace = f"cuda-{i}" + lease = cache.lookup_and_pin(namespace, tokens, 0) + reservation = controller.try_reserve_output(execution, rows, 0, 0) + self.assertIsNotNone(reservation) + plans.append(controller.freeze_plan(execution, namespace, tokens, rows, 0, + lease, reservation)) + cpu = ((torch.arange(rows * 16).reshape(rows, 16) % 113) + i).to(torch.bfloat16) + expected.append({"aux": cpu.clone(), "last_hidden": cpu + 5}) + values = {name: tensor.to("cuda") for name, tensor in expected[-1].items()} + producer = torch.cuda.Event() + producer.record(torch.cuda.current_stream()) + spans.append(CapturedSpan(execution, 0, rows, values, producer)) + # Each worker must enqueue its own real D2H before either completion + # observation is allowed to return. This exposes overlapping ownership + # even on devices whose physical DMA engines serialize the transfers. + submitted = threading.Barrier(3) + finish = threading.Event() + real_event = torch.cuda.Event + class CopyEvent: + def __init__(self): + self.event = real_event() + def record(self, stream): + self.event.record(stream) + def synchronize(self): + submitted.wait(10) + if not finish.wait(10): + raise RuntimeError("CUDA test host gate did not open") + self.event.synchronize() + futures = [] + try: + with mock.patch("specforge_sglang_capture.hs.controller.torch.cuda.Event", CopyEvent): + futures = [controller.submit(plan, span) for plan, span in zip(plans, spans)] + spans.clear() + values = None + submitted.wait(10) + self.assertTrue(all(not future.done() for future in futures)) + self.assertEqual(controller.stats()["pinned_bytes"], sample_bytes * 2) + offsets = [controller._reservations[p.reservation.reservation_id].pinned_offset + for p in plans] + self.assertEqual(abs(offsets[1] - offsets[0]), sample_bytes) + extra = ExecutionIdentity("cuda-probe", "extra", "extra", 0) + self.assertIsNone(controller.try_reserve_output(extra, 4, 0, 0)) + drain = None + if cancel_first: + drain = controller.invalidate(plans[0].execution, "real DMA cancellation") + self.assertFalse(drain.done()) + self.assertEqual(controller.stats()["pinned_bytes"], sample_bytes * 2) + finish.set() + for i, future in enumerate(futures): + if cancel_first and i == 0: + with self.assertRaises(CaptureCancelledError): + future.result(10) + else: + sample = future.result(10) + samples.append(sample) + for name, tensor in sample.features.items(): + self.assertTrue(torch.equal(tensor, expected[i][name])) + if drain is not None: + self.assertTrue(drain.wait(2)) + self.assertEqual(controller.stats()["pinned_bytes"], 0) + self.assertEqual(controller.stats()["device_bytes"], 0) + for sample in samples: + sample.release() + self.assertEqual(cache.stats()["ready_blocks"], 16 * (1 if cancel_first else 2)) + stats = controller.stats() + self.assertEqual(stats["output_bytes"], 0) + self.assertEqual(stats["quarantined_executions"], 0) + self.assertEqual(stats["pinned_arena_bytes"], sample_bytes * 2) + self.assertEqual(stats["pinned_free_bytes"], sample_bytes * 2) + print(json.dumps({"real_cuda": True, "cancel_first": cancel_first, + "concurrently_submitted_d2h": 2, "bitexact": True, + "stats": stats})) + finally: + finish.set() + for future in futures: + try: + future.result(15).release() + except Exception: + pass + for sample in samples: + sample.release() + controller.poll_completed() + controller.poll_drained() + controller.shutdown() + + def test_two_real_d2h_jobs_are_isolated_and_bitexact(self): + self.run_pair(False) + + def test_cancelled_real_d2h_slot_stays_owned_until_observed_drain(self): + self.run_pair(True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_pipeline_diagnostics.py b/tests/test_runtime/test_hs_pipeline_diagnostics.py new file mode 100644 index 0000000000..5df18e496c --- /dev/null +++ b/tests/test_runtime/test_hs_pipeline_diagnostics.py @@ -0,0 +1,106 @@ +"""CPU timing-boundary tests; these do not claim CUDA/RDMA performance.""" +import os +import threading +import unittest +from unittest import mock + +import torch + +from tests.test_runtime import test_hs_plugin as fixture + +_Store = fixture._Store +from specforge_sglang_capture.sink import SpecCaptureSink + + +class PipelineDiagnosticsTests(unittest.TestCase): + setUp = fixture.HSPluginTests.setUp + tearDown = fixture.HSPluginTests.tearDown + runtime = fixture.HSPluginTests.runtime + request = fixture.HSPluginTests.request + admit = fixture.HSPluginTests.admit + forward = fixture.HSPluginTests.forward + spin = fixture.HSPluginTests.spin + + def test_default_off_preserves_response_and_does_not_record(self): + with mock.patch.dict(os.environ, {"SPECFORGE_HS_DIAGNOSTICS": "0"}): + hs = self.runtime() + req = self.request(hs) + self.admit(hs, req) + output = self.forward(hs, [req], [8]) + self.spin(hs, lambda: hs.deferred[0][1].done()) + hs.deferred[0][1].result() + self.assertIsNone(output.hs_jobs[0].diagnostic) + self.assertFalse(hs.diagnostics) + result = hs.capture.results[req.rid] + self.assertFalse(any(k.startswith("p0_v1_") for k in result["timings"])) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + + def test_sampled_boundaries_preserve_writer_drain_and_bounded_records(self): + gate = threading.Event() + store = _Store(gate=gate) + with mock.patch.dict(os.environ, {"SPECFORGE_HS_DIAGNOSTICS": "1", + "SPECFORGE_HS_DIAGNOSTICS_EVERY": "1", "SPECFORGE_HS_DIAGNOSTICS_MAX": "1"}): + hs = self.runtime(store=store) + reqs = [self.request(hs, name) for name in ("a", "b")] + for req in reqs: + self.admit(hs, req) + output = self.forward(hs, reqs, [8, 8], wait_materialize=True) + first, second = output.hs_jobs + self.assertIs(first.sink, second.sink) + self.assertTrue(store.entered.wait(2)) + try: + self.assertFalse(first.sink.done()) + self.assertFalse(first.output.done()) + self.assertGreater(hs.controller.stats()["output_bytes"], 0) + self.assertTrue(store.registered) + self.assertNotIn("p0_v1_sink_done_ns", first.sink.p0_v1_timings) + finally: + gate.set() + first.sink.result(timeout=3) + self.assertFalse(store.registered) + self.assertFalse(first.output.done()) # Native owner must still observe. + hs.poll() + for _, completion in hs.deferred: + completion.result() + diag, = hs.diagnostics + self.assertIs(diag, first.diagnostic) + self.assertIsNone(second.diagnostic) + keys = ["p0_v1_sink_enqueue_ns", "p0_v1_sink_start_ns", "p0_v1_sink_done_ns", + "p0_v1_owner_observed_ns", "p0_v1_response_callback_ns"] + self.assertEqual([diag[k] for k in keys], sorted(diag[k] for k in keys)) + self.assertLessEqual(diag["p0_v1_materialize_done_ns"], diag["p0_v1_runtime_sink_enqueue_ns"]) + self.assertGreaterEqual(diag["p0_v1_materialize_to_enqueue_ms"], 0) + self.assertEqual(diag["p0_v1_sink_batch_samples"], 2) + self.assertEqual(diag["p0_v1_consume_batch_samples"], 2) + for name in ("register", "put", "unregister", "queue"): + self.assertGreaterEqual(diag["p0_v1_sink_" + name + "_ms"], 0) + self.assertGreater(diag["p0_v1_sink_registered_bytes"], 0) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + + def test_sink_queue_is_before_writer_start_and_metadata_has_no_tensors(self): + with mock.patch.dict(os.environ, {"SPECFORGE_HS_DIAGNOSTICS": "1"}): + writer = SpecCaptureSink() + gate, started = threading.Event(), threading.Event() + self.addCleanup(writer.close) + def block(): + started.set() + self.assertTrue(gate.wait(3)) + blocker = writer._executor.submit(block) + self.assertTrue(started.wait(2)) + writer._store = _Store() + spec = {"store_id": "test", "sample_id": "a", "features": {"aux": "hidden"}} + future = writer.submit_samples([(spec, torch.ones(2, 2), None)]) + try: + self.assertNotIn("p0_v1_sink_start_ns", future.p0_v1_timings) + finally: + gate.set() + blocker.result(timeout=3) + future.result(timeout=3) + d = future.p0_v1_timings + self.assertLessEqual(d["p0_v1_sink_enqueue_ns"], d["p0_v1_sink_start_ns"]) + self.assertLessEqual(d["p0_v1_sink_start_ns"], d["p0_v1_sink_done_ns"]) + self.assertTrue(all(type(value) in (int, float, bool) for value in d.values())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_plugin.py b/tests/test_runtime/test_hs_plugin.py new file mode 100644 index 0000000000..f901b314e7 --- /dev/null +++ b/tests/test_runtime/test_hs_plugin.py @@ -0,0 +1,322 @@ +"""CPU native-output contracts; real DMA/model/TP validation remains separate.""" + +import ctypes +import dataclasses +import json +import os +import sys +import threading +import time +import unittest +from concurrent.futures import Future +from types import SimpleNamespace +from unittest import mock + +import torch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "plugins", "sglang-spec-capture")) + +from specforge_sglang_capture import capture, sink +from specforge_sglang_capture.hs.attempts import AttemptCapacityDeclined, AttemptRegistry, digest_request +from specforge_sglang_capture.hs.config import HSConfig +from specforge_sglang_capture.hs.protocol import FeatureSchema, FeatureSpec +from specforge_sglang_capture.hs.runtime import HSCaptureRuntime +from specforge.inference.capture_protocol import CaptureCapacityDeclined, CaptureProtocol + + +@dataclasses.dataclass +class _Proposal: + max_prefix_len: int + identity: tuple + opaque: object = None + + +class _Store: + def __init__(self, gate=None, status=0): + self.values, self.registered = {}, set() + self.gate, self.status = gate, status + self.entered = threading.Event() + + def register_buffer(self, ptr, size): + self.registered.add(ptr) + return 0 + + def unregister_buffer(self, ptr): + self.registered.remove(ptr) + return 0 + + def batch_put_from(self, keys, pointers, sizes, config): + self.entered.set() + if self.gate is not None and not self.gate.wait(5): + raise TimeoutError("test writer gate did not open") + for key, ptr, size in zip(keys, pointers, sizes): + self.values[key] = ctypes.string_at(ptr, size) + return [self.status] * len(keys) + + def remove(self, key, *, force=False): + self.values.pop(key, None) + return 0 + + +def _schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)), + FeatureSpec("last_hidden", 2, "float32", ()))) + + +class HSPluginTests(unittest.TestCase): + def setUp(self): + self.env = mock.patch.dict(os.environ, {"SGLANG_SPEC_CAPTURE_GPU_PUT": "0", "MOONCAKE_PROTOCOL": "tcp"}) + self.env.start() + self.protocol_module = mock.patch.dict(sys.modules, { + "sglang.srt.managers.prefix_admission": SimpleNamespace(PrefixAdmissionProposal=_Proposal)}) + self.protocol_module.start() + self.runtimes = [] + + def tearDown(self): + for runtime in self.runtimes: + if runtime.quarantined_publications or runtime.quarantined_snapshots: + runtime.capture.sink._executor.shutdown(wait=True) + runtime.controller._executor.shutdown(wait=True) + else: + runtime.shutdown() + self.protocol_module.stop() + self.env.stop() + + def runtime(self, *, store=None, cache=True, capacity=16, parallel=None): + writer = sink.SpecCaptureSink([1, 3]) + writer._store, writer._put_config = store or _Store(), object() + deferred = [] + native = capture.CaptureRuntime(sink=writer, is_writer=True, gpu_put=False, + max_pending_batches=2, defer_response=lambda req, completion: deferred.append((req, completion))) + scheduler = SimpleNamespace(abort_request=lambda req: None) + config = HSConfig(capture=True, cache=cache, chunk_tokens=4, cache_bytes=4096, + pinned_bytes=4096, output_bytes=8192, device_bytes=4096, attempt_capacity=capacity) + kwargs = dict(manifest={"model": "cpu-fixture", "weights": "load-epoch"}, schema=_schema()) + if parallel is None: + hs = HSCaptureRuntime(scheduler, native, config, tp_size=1, **kwargs) + else: + getter = mock.Mock(return_value=parallel) + with mock.patch.dict(sys.modules, {"sglang.srt.runtime_context": SimpleNamespace(get_parallel=getter)}): + hs = HSCaptureRuntime(scheduler, native, config, **kwargs) + getter.assert_called_once_with() + native.hs = hs + hs.deferred = deferred + self.runtimes.append(hs) + return hs + + def request(self, hs, sample="s", *, tokens=tuple(range(8)), mask=1): + spec = {"store_id": "store", "sample_id": sample, "gen": 1, "replace": True, + "features": {"aux": "hidden_states", "last_hidden": "target"}, + "passthrough": [{"name": "loss_mask", "data": [[mask] * len(tokens)], + "shape": [1, len(tokens)], "dtype": "int64"}]} + digest = digest_request(tokens, spec) + body = {"protocol_version": 1, "action": "begin", "server_epoch": hs.epoch, + "store_id": "store", "sample_id": sample, "gen": 1, + "attempt_id": "attempt-" + sample, "request_digest": digest, + "publication_ownership": "attempt-v1"} + reply = hs.control(hs.s, body) + spec.update({key: reply[key] for key in ("server_epoch", "attempt_id", "reservation_token", "object_key_layout")}) + spec.update(protocol_version=1, capture_namespace=hs.namespace, + capture_signature=hs.signature, request_digest=digest) + return SimpleNamespace(rid="req-" + sample, origin_input_ids=list(tokens), + sampling_params=SimpleNamespace(custom_params={"spec_capture": json.dumps(spec)}, max_new_tokens=0), + finished=lambda: True) + + def admit(self, hs, req, lawful=None): + proposal = hs.prepare(hs.s, req) + L = proposal.max_prefix_len if lawful is None else lawful + hs.after_match(hs.s, req, proposal, L) + self.assertTrue(hs.reserve(hs.s, req, proposal, L)) + hs.commit(hs.s, req, proposal, L) + return proposal, L + + def forward(self, hs, reqs, lengths, *, wait_materialize=False): + aux, last, prefixes = [], [], [] + for req, length in zip(reqs, lengths): + L = req._specforge_hs_plan.L + rows = torch.arange(L * 2, (L + length) * 2, dtype=torch.float32).view(length, 2) + aux.append(rows) + last.append(rows + 1000) + prefixes.append(L) + batch = SimpleNamespace(reqs=reqs) + forward = SimpleNamespace(extend_seq_lens_cpu=lengths, extend_prefix_lens_cpu=prefixes, + forward_mode=SimpleNamespace(is_extend=lambda: True)) + logits = SimpleNamespace(hidden_states=torch.cat(aux), last_hidden_states=torch.cat(last), + hidden_states_token_indices=None) + output = hs.capture.observer.after_forward(batch, forward, logits, can_run_graph=True) + with mock.patch.object(torch.Tensor, "to", side_effect=AssertionError("native D2H is duplicated")): + host = output.copy_to_host(lambda tensor: self.fail("HS must not call native copy_tensor")) + if wait_materialize: + self.spin(hs, lambda: all(job.materialize.done() for job in output.hs_jobs)) + host.consume(batch, [None] * len(reqs)) + return output + + def spin(self, hs, condition): + deadline = time.monotonic() + 5 + while not condition(): + hs.poll() + if time.monotonic() > deadline: + self.fail("HS pipeline did not make progress") + time.sleep(0.001) + hs.poll() + + def test_native_complete_n_rows_from_nonaligned_prefix_and_loss_mask_independent_identity(self): + hs = self.runtime(parallel=SimpleNamespace(tp_size=1, tp_rank=0)) + first = self.request(hs, "cold") + self.admit(hs, first) + self.forward(hs, [first], [8]) + self.spin(hs, lambda: hs.deferred[0][1].done()) + hs.deferred[0][1].result() + cold = hs.capture.results[first.rid] + self.assertEqual(cold["state"], "PUBLISHED") + self.assertEqual(cold["rank"], hs.tp_rank) + self.assertEqual(cold["publication_certificate"]["rank"], hs.tp_rank) + self.assertTrue(cold["writer_drained"]) + self.assertEqual(cold["request_digest"], digest_request(first.origin_input_ids, capture.request_spec(first))) + second = self.request(hs, "warm", mask=0) + proposal, L = self.admit(hs, second, lawful=3) + self.assertEqual(proposal.max_prefix_len, 4) + self.assertEqual(L, 3) + self.forward(hs, [second], [5]) + self.spin(hs, lambda: hs.deferred[1][1].done()) + hs.deferred[1][1].result() + warm = hs.capture.results[second.rid] + self.assertEqual((warm["N"], warm["L"], warm["span"]), (8, 3, [3, 8])) + self.assertNotEqual(cold["request_digest"], warm["request_digest"]) + values = hs.capture.sink._store.values + for name in ("hidden_states", "target"): + self.assertEqual(values[cold["physical_keys"][name]], values[warm["physical_keys"][name]]) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + self.assertGreater(hs.cache.stats()["cache_owned_bytes"], 0) + + def test_cancel_waits_for_shared_sink_and_cannot_delete_sibling_attempt(self): + gate = threading.Event() + store = _Store(gate) + hs = self.runtime(store=store) + reqs = [self.request(hs, name) for name in ("cancel", "sibling")] + for req in reqs: + self.admit(hs, req) + output = self.forward(hs, reqs, [8, 8], wait_materialize=True) + first, sibling = output.hs_jobs + self.assertIs(first.sink, sibling.sink) + self.assertIsNotNone(first.sink) + self.assertEqual((first.sink_count, sibling.sink_count), (2, 2)) + self.spin(hs, store.entered.is_set) + held = first.sample + hs.deferred[0][1].cancel() + self.assertTrue(hs.deferred[0][1].done()) + self.assertIs(first.sample, held) + self.assertFalse(first.sink.done()) + self.assertGreater(hs.controller.stats()["output_bytes"], 0) + gate.set() + self.spin(hs, lambda: not hs.jobs) + hs.deferred[1][1].result() + result = hs.capture.results[reqs[1].rid] + self.assertTrue(all(key in store.values for key in result["physical_keys"].values())) + cancelled = capture.request_spec(reqs[0]) + self.assertFalse(any(cancelled["reservation_token"] in key for key in store.values)) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + + def test_unknown_native_write_retains_sources_registration_and_output_charge(self): + store = _Store(status=-800) + hs = self.runtime(store=store) + req = self.request(hs) + self.admit(hs, req) + self.forward(hs, [req], [8]) + self.spin(hs, lambda: bool(hs.quarantined_publications)) + self.assertGreater(hs.controller.stats()["output_bytes"], 0) + self.assertTrue(store.registered) + self.assertFalse(req._specforge_hs_attempt.transfer_certificate) + self.assertTrue(hs.capture.sink._uncertain_publications) + with self.assertRaises(RuntimeError): + hs.shutdown() + + def test_cache_off_preserves_attempt_keys_full_rows_and_zero_reuse(self): + hs = self.runtime(cache=False) + for sample in ("a", "b"): + req = self.request(hs, sample) + proposal, _ = self.admit(hs, req) + self.assertEqual(proposal.max_prefix_len, 0) + self.forward(hs, [req], [8]) + self.spin(hs, lambda: hs.deferred[-1][1].done()) + hs.deferred[-1][1].result() + self.assertEqual(hs.capture.results[req.rid]["span"], [0, 8]) + self.assertEqual(hs.capture.results[req.rid]["rank"], hs.tp_rank) + self.assertFalse(hs.capability()["cache_enabled"]) + self.assertEqual(hs.cache.stats()["cache_owned_bytes"], 0) + + def test_rollback_before_and_after_freeze_releases_only_its_candidate(self): + hs = self.runtime() + req = self.request(hs) + proposal = hs.prepare(hs.s, req) + hs.after_match(hs.s, req, proposal, 0) + self.assertTrue(hs.reserve(hs.s, req, proposal, 0)) + hs.rollback(hs.s, req, proposal, "native-batch-allocation-failed") + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + self.assertEqual(hs.cache.stats()["active_leases"], 0) + next_proposal, _ = self.admit(hs, req) + self.assertGreater(next_proposal.identity[3], proposal.identity[3]) + hs.rollback(hs.s, req, next_proposal, "peer-commit-failed") + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + self.assertEqual(hs.cache.stats()["active_leases"], 0) + + def test_owner_thread_and_digest_fail_closed_before_reservation(self): + hs = self.runtime() + req = self.request(hs) + req.origin_input_ids[0] = 99 + with self.assertRaises(ValueError): + hs.prepare(hs.s, req) + self.assertEqual(hs.controller.stats()["output_bytes"], 0) + errors = [] + thread = threading.Thread(target=lambda: self._record_error(errors, hs)) + thread.start() + thread.join() + self.assertEqual(len(errors), 1) + self.assertIn("scheduler thread", str(errors[0])) + + @staticmethod + def _record_error(errors, hs): + try: + hs.control(hs.s, {"protocol_version": 1, "action": "capability"}) + except Exception as exc: + errors.append(exc) + + def test_begin_batch_capacity_is_atomic_and_exposes_only_ungranted_backpressure(self): + hs = self.runtime(capacity=2) + self.request(hs, "held") + task = lambda name: {"server_epoch": hs.epoch, "store_id": "store", "sample_id": name, + "gen": 1, "attempt_id": "a-" + name, "request_digest": "digest", + "publication_ownership": "attempt-v1"} + before = dict(hs.registry.active) + reply = hs.control(hs.s, {"protocol_version": 1, "action": "begin_batch", + "server_epoch": hs.epoch, "tasks": [task("new1"), task("new2")]}) + self.assertEqual(reply["error_code"], "capture_capacity_declined") + self.assertIs(reply["reservation_granted"], False) + self.assertEqual(hs.registry.active, before) + protocol = CaptureProtocol("http://fixture", lambda *args, **kwargs: reply, 1) + with self.assertRaises(CaptureCapacityDeclined): + protocol.call({"action": "begin_batch"}) + with self.assertRaises(RuntimeError) as failure: + protocol.call({"action": "resolve"}) + self.assertNotIsInstance(failure.exception, CaptureCapacityDeclined) + + def test_old_attempt_cleanup_never_targets_new_server_keys(self): + store = _Store() + first, second = self.runtime(store=store), self.runtime(store=store) + results = [] + specs = [] + for hs in (first, second): + req = self.request(hs, "same-sample") + specs.append(capture.request_spec(req)) + self.admit(hs, req) + self.forward(hs, [req], [8]) + self.spin(hs, lambda: hs.deferred[0][1].done()) + hs.deferred[0][1].result() + results.append(hs.capture.results[req.rid]) + first.capture.sink.discard_sample(specs[0]).result(5) + self.assertTrue(all(key in store.values for key in results[1]["physical_keys"].values())) + self.assertTrue(all(key not in store.values for key in results[0]["physical_keys"].values())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_runtime.py b/tests/test_runtime/test_hs_runtime.py new file mode 100644 index 0000000000..adb8601a3c --- /dev/null +++ b/tests/test_runtime/test_hs_runtime.py @@ -0,0 +1,779 @@ +"""CPU-only adversarial tests of the production Plan A cache/controller.""" + +import gc +import threading +import weakref +from contextlib import nullcontext +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from unittest import mock + +import torch + +from specforge_sglang_capture.hs import ( + AdmissionBudgetError, CaptureCancelledError, CaptureCompletion, + CaptureContractError, CaptureController, CapturedSpan, ExecutionIdentity, + FeatureSchema, FeatureSpec, PrefixFeatureCache, +) + + +def schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)), + FeatureSpec("last_hidden", 2, "float32", ()))) + + +def identity(n=0, epoch=0): + return ExecutionIdentity("server", f"request-{n}", f"attempt-{n}", epoch) + + +def features(n, start=0): + rows = torch.arange(start * 2, (start + n) * 2, dtype=torch.float32).reshape(n, 2) + return {"aux": rows.clone(), "last_hidden": (rows + 1000).clone()} + + +def populate(cache, tokens, values=None, namespace="ns"): + values = features(len(tokens)) if values is None else values + for key in cache.block_keys(namespace, tokens): + fill = cache.reserve_fill(key, identity()) + if fill is not None: + cache.publish_ready(fill, {k: v[key.start:key.end] for k, v in values.items()}) + + +class GateEvent: + def __init__(self): + self.entered = threading.Event() + self.finished = threading.Event() + + def synchronize(self): + self.entered.set() + if not self.finished.wait(5): + raise RuntimeError("test producer event was not completed") + + +class CacheTests(unittest.TestCase): + def setUp(self): + self.cache = PrefixFeatureCache(schema(), 4, 128) + self.tokens = tuple(range(8)) + + def test_schema_order_and_shape_are_semantic(self): + a = FeatureSchema((FeatureSpec("aux", 2, "float32", (3, 1)),)) + b = FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)),)) + self.assertNotEqual(a.signature, b.signature) + self.assertEqual(schema().row_bytes, 16) + for kwargs in [dict(name="unknown", width=2, dtype="float32", layer_ids=()), + dict(name="aux", width=2, dtype="torch.float32", layer_ids=(1,)), + dict(name="last_hidden", width=2, dtype="float32", layer_ids=(1,))]: + with self.assertRaises(CaptureContractError): + FeatureSpec(**kwargs) + with self.assertRaises(CaptureContractError): + schema().validate({"aux": torch.zeros(4, 2)}, 4) + + def test_keys_bind_entire_prefix_namespace_and_chunk_size(self): + same_tail = (9, 10, 11, 12) + self.tokens[4:] + keys = self.cache.block_keys("ns", self.tokens) + self.assertEqual(len(keys[0].digest), 64) + self.assertNotEqual(keys[1].digest, self.cache.block_keys("ns", same_tail)[1].digest) + self.assertNotEqual(keys[0].digest, self.cache.block_keys("other", self.tokens)[0].digest) + self.assertNotEqual(keys[0].digest, PrefixFeatureCache(schema(), 2, 128).block_keys("ns", self.tokens)[0].digest) + self.assertEqual(keys, self.cache.block_keys("ns", self.tokens)) + for tokens in [(True, 1, 2, 3), (-1, 1, 2, 3), (2**63, 1, 2, 3)]: + with self.assertRaises(CaptureContractError): + self.cache.block_keys("ns", tokens) + with self.assertRaises(CaptureContractError): + self.cache.reserve_fill(replace(keys[0], _tokens=(9, 9, 9, 9)), identity()) + + def test_hole_and_writing_stop_contiguous_coverage(self): + first, second = self.cache.block_keys("ns", self.tokens) + pending = self.cache.reserve_fill(first, identity()) + later = self.cache.reserve_fill(second, identity(1)) + self.cache.publish_ready(later, features(4, 4)) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + self.assertEqual(lease.coverage_end, 0) + self.cache.release_lease(lease) + self.cache.publish_ready(pending, features(4)) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + self.assertEqual(lease.coverage_end, 8) + self.cache.release_lease(lease) + + def test_no_incomplete_tail_or_limit_chunk_reuse(self): + populate(self.cache, self.tokens) + for query, limit, expected in [(self.tokens, 8, 8), (self.tokens, 7, 4), + (self.tokens[:7], 7, 4), (self.tokens[:3], 3, 0)]: + lease = self.cache.lookup_and_pin("ns", query, limit) + self.assertEqual(lease.coverage_end, expected) + self.cache.release_lease(lease) + + def test_missing_feature_publish_rolls_back_without_ready(self): + fill = self.cache.reserve_fill(self.cache.block_keys("ns", self.tokens)[0], identity()) + with self.assertRaises(CaptureContractError): + self.cache.publish_ready(fill, {"aux": torch.zeros(4, 2)}) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.cache.stats()["fill_reservations"], 0) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + self.assertEqual(lease.coverage_end, 0) + self.cache.release_lease(lease) + + def test_shrink_inside_chunk_consumes_handle_and_stale_release_is_safe(self): + populate(self.cache, self.tokens) + old = self.cache.lookup_and_pin("ns", self.tokens, 8) + new = self.cache.shrink_lease(old, 3) + self.cache.release_lease(old) + self.assertEqual(self.cache.stats()["active_leases"], 1) + self.assertEqual(self.cache.stats()["leased_blocks"], 1) + out = features(8, 100) + self.cache.copy_prefix_into(new, 3, out) + self.assertTrue(torch.equal(out["aux"][:3], features(3)["aux"])) + with self.assertRaises(CaptureContractError): + self.cache.copy_prefix_into(old, 3, out) + self.cache.release_lease(new) + self.cache.release_lease(new) + self.assertEqual(self.cache.stats()["active_leases"], 0) + + def test_pinned_blocks_cannot_be_evicted_and_cache_does_not_alias(self): + values = features(8) + populate(self.cache, self.tokens, values) + values["aux"].fill_(-99) + lease = self.cache.lookup_and_pin("ns", self.tokens, 8) + key = self.cache.block_keys("ns", (99, 98, 97, 96))[0] + self.assertIsNone(self.cache.reserve_fill(key, identity(1))) + out = features(8) + self.cache.copy_prefix_into(lease, 8, out) + self.assertTrue(torch.equal(out["aux"], features(8)["aux"])) + out["aux"].fill_(-999) + self.cache.copy_prefix_into(lease, 8, out) + self.assertTrue(torch.equal(out["aux"], features(8)["aux"])) + self.cache.release_lease(lease) + fill = self.cache.reserve_fill(key, identity(1)) + self.assertIsNotNone(fill) + self.assertLessEqual(self.cache.stats()["cache_bytes"], 128) + self.cache.abort_fill(fill) + + def test_single_committer_and_foreign_abort_do_not_erase_ready(self): + key = self.cache.block_keys("ns", self.tokens)[0] + with ThreadPoolExecutor(8) as pool: + results = list(pool.map(lambda n: self.cache.reserve_fill(key, identity(n)), range(8))) + winners = [r for r in results if r is not None] + self.assertEqual(len(winners), 1) + owner = winners[0] + self.cache.abort_fill(replace(owner)) + self.assertEqual(self.cache.stats()["writing_blocks"], 1) + self.assertTrue(self.cache.publish_ready(owner, features(4))) + self.cache.abort_fill(owner) + self.assertFalse(self.cache.publish_ready(owner, features(4))) + self.assertEqual(self.cache.stats()["ready_blocks"], 1) + + def test_lookup_pin_is_atomic_against_eviction(self): + cache = PrefixFeatureCache(schema(), 4, 64) + populate(cache, self.tokens[:4]) + entered, resume = threading.Event(), threading.Event() + complete = cache._complete + def blocked(entry): + entered.set() + self.assertTrue(resume.wait(3)) + return complete(entry) + with mock.patch.object(cache, "_complete", blocked), ThreadPoolExecutor(2) as pool: + lookup = pool.submit(cache.lookup_and_pin, "ns", self.tokens[:4], 4) + self.assertTrue(entered.wait(3)) + key = cache.block_keys("ns", (8, 9, 10, 11))[0] + eviction = pool.submit(cache.reserve_fill, key, identity(2)) + resume.set() + lease = lookup.result(3) + self.assertEqual(lease.coverage_end, 4) + self.assertIsNone(eviction.result(3)) + cache.release_lease(lease) + + def test_abort_during_cpu_writer_keeps_reservation_until_drain(self): + fill = self.cache.reserve_fill(self.cache.block_keys("ns", self.tokens)[0], identity()) + entered, resume = threading.Event(), threading.Event() + real_empty = torch.empty + def blocked(*args, **kwargs): + entered.set() + self.assertTrue(resume.wait(3)) + return real_empty(*args, **kwargs) + values = features(4) + with mock.patch("specforge_sglang_capture.hs.cache.torch.empty", blocked), ThreadPoolExecutor(1) as pool: + writer = pool.submit(self.cache.publish_ready, fill, values) + self.assertTrue(entered.wait(3)) + self.cache.abort_fill(fill) + self.assertEqual(self.cache.stats()["cache_bytes"], 64) + self.assertEqual(self.cache.stats()["fill_reservations"], 1) + resume.set() + self.assertFalse(writer.result(3)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + + def test_release_during_cpu_reader_defers_reclaim(self): + cache = PrefixFeatureCache(schema(), 4, 64) + populate(cache, self.tokens[:4]) + lease = cache.lookup_and_pin("ns", self.tokens[:4], 4) + dest = features(4) + entered, resume = threading.Event(), threading.Event() + real_copy = torch.Tensor.copy_ + def blocked(tensor, *args, **kwargs): + entered.set() + self.assertTrue(resume.wait(3)) + return real_copy(tensor, *args, **kwargs) + with mock.patch.object(torch.Tensor, "copy_", blocked), ThreadPoolExecutor(1) as pool: + reader = pool.submit(cache.copy_prefix_into, lease, 4, dest) + self.assertTrue(entered.wait(3)) + cache.release_lease(lease) + key = cache.block_keys("ns", (8, 9, 10, 11))[0] + self.assertIsNone(cache.reserve_fill(key, identity(2))) + resume.set() + reader.result(3) + fill = cache.reserve_fill(key, identity(2)) + self.assertIsNotNone(fill) + cache.abort_fill(fill) + self.assertEqual(cache.stats()["active_leases"], 0) + + + def test_failed_cache_copy_does_not_retain_owned_storage_via_traceback(self): + fill = self.cache.reserve_fill(self.cache.block_keys("ns", self.tokens)[0], identity()) + values = features(4) + real_empty, real_copy = torch.empty, torch.Tensor.copy_ + allocated, retained_errors = [], [] + count = [0] + def allocate(*args, **kwargs): + tensor = real_empty(*args, **kwargs) + allocated.append(weakref.ref(tensor)) + return tensor + def fail_second(tensor, *args, **kwargs): + count[0] += 1 + if count[0] == 2: + raise RuntimeError(tensor) # Deliberately retain the tensor in args/traceback. + return real_copy(tensor, *args, **kwargs) + with mock.patch("specforge_sglang_capture.hs.cache.torch.empty", allocate), \ + mock.patch.object(torch.Tensor, "copy_", fail_second): + try: + self.cache.publish_ready(fill, values) + except RuntimeError as error: + retained_errors.append(error) + gc.collect() + self.assertEqual(len(retained_errors), 1) + self.assertTrue(all(ref() is None for ref in allocated)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + + def test_owner_group_charges_entire_storage_and_any_lease_blocks_eviction(self): + cache = PrefixFeatureCache(schema(), 4, 160) + tokens = tuple(range(9)) + values = features(9) + self.assertTrue(cache._adopt_released_sample(cache.block_keys("ns", tokens), values)) + values.clear() # Ownership has transferred; callers retain no aliases. + self.assertEqual(cache.stats()["cache_bytes"], 144) + self.assertEqual(cache.stats()["cache_logical_bytes"], 128) + lease = cache.lookup_and_pin("ns", tokens, 4) + different = cache.block_keys("other", (9, 8, 7, 6))[0] + self.assertIsNone(cache.reserve_fill(different, identity(2))) + self.assertEqual(cache.stats()["ready_blocks"], 2) + self.assertEqual(cache.stats()["cache_bytes"], 144) + out = features(4, 100) + cache.copy_prefix_into(lease, 4, out) + self.assertTrue(torch.equal(out["aux"], features(4)["aux"])) + cache.release_lease(lease) + fill = cache.reserve_fill(different, identity(2)) + self.assertIsNotNone(fill) + self.assertEqual(cache.stats()["ready_blocks"], 0) + self.assertEqual(cache.stats()["cache_owned_bytes"], 0) + self.assertEqual(cache.stats()["cache_bytes"], 64) + cache.abort_fill(fill) + + def test_contiguous_owner_prefix_uses_one_copy_per_feature(self): + cache = PrefixFeatureCache(schema(), 4, 160) + tokens = tuple(range(9)) + values = features(9) + cache._adopt_released_sample(cache.block_keys("ns", tokens), values) + values.clear() + lease = cache.lookup_and_pin("ns", tokens, 8) + out = features(8, 100) + copies = [] + real_copy = torch.Tensor.copy_ + def copy(target, source, *args, **kwargs): + copies.append(tuple(target.shape)) + return real_copy(target, source, *args, **kwargs) + with mock.patch.object(torch.Tensor, "copy_", copy): + cache.copy_prefix_into(lease, 8, out) + cache.release_lease(lease) + self.assertEqual(copies, [(8, 2), (8, 2)]) + self.assertTrue(torch.equal(out["aux"], features(8)["aux"])) + + def test_adoption_counts_hidden_backing_storage_and_rejects_oversize(self): + values = {name: tensor[:4] for name, tensor in features(9).items()} + keys = self.cache.block_keys("ns", self.tokens[:4]) + self.assertFalse(self.cache._adopt_released_sample(keys, values)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + larger = PrefixFeatureCache(schema(), 4, 160) + self.assertTrue(larger._adopt_released_sample(keys, values)) + self.assertEqual(larger.stats()["cache_bytes"], 144) + self.assertEqual(larger.stats()["cache_logical_bytes"], 64) + + + +class ControllerTests(unittest.TestCase): + def setUp(self): + self.cache = PrefixFeatureCache(schema(), 4, 1024) + self.controller = CaptureController(self.cache, schema(), 512, 512, 512) + self.samples = [] + + def tearDown(self): + for sample in self.samples: + sample.release() + self.controller.poll_completed() + self.controller.poll_drained() + self.controller.shutdown() + + def plan(self, n=8, L=0, execution=None, namespace="ns", passthrough=0): + execution = identity() if execution is None else execution + tokens = tuple(range(n)) + lease = self.cache.lookup_and_pin(namespace, tokens, n) + self.assertGreaterEqual(lease.coverage_end, L) + lease = self.cache.shrink_lease(lease, L) + reservation = self.controller.try_reserve_output(execution, n, L, passthrough) + self.assertIsNotNone(reservation) + return self.controller.freeze_plan(execution, namespace, tokens, n, L, lease, reservation) + + def submit(self, plan, event=None): + span = CapturedSpan(plan.execution, plan.L, plan.N, features(plan.N-plan.L, plan.L), event) + return self.controller.submit(plan, span) + + def take(self, future): + sample = future.result(5) + self.samples.append(sample) + return sample + + def test_full_sample_from_non_aligned_prefix_and_suffix(self): + cold = self.take(self.submit(self.plan(n=9))) + self.assertTrue(torch.equal(cold.features["aux"], features(9)["aux"])) + self.assertEqual(self.cache.stats()["ready_blocks"], 0) + cold.release() + self.assertEqual(self.cache.stats()["ready_blocks"], 2) + self.controller.poll_completed() + self.controller.poll_drained() + plan = self.plan(n=9, L=5, execution=identity(1)) + warm = self.take(self.submit(plan)) + for name in warm.features: + self.assertTrue(torch.equal(warm.features[name], features(9)[name])) + warm.features["aux"].fill_(-99) + lease = self.cache.lookup_and_pin("ns", tuple(range(9)), 9) + out = features(9) + self.cache.copy_prefix_into(lease, 8, out) + self.cache.release_lease(lease) + self.assertTrue(torch.equal(out["aux"][:8], features(8)["aux"])) + self.assertEqual(self.controller.stats()["active_leases"], 0) + + def test_output_charge_survives_completion_and_cancel_until_sink_release(self): + self.controller = CaptureController(self.cache, schema(), 512, 128, 512) + sample = self.take(self.submit(self.plan())) + self.assertEqual(self.controller.stats()["output_bytes"], 128) + self.assertEqual(self.controller.stats()["pinned_bytes"], 0) + self.assertIsNone(self.controller.try_reserve_output(identity(1), 8, 0, 0)) + drain = self.controller.invalidate(identity(), "HTTP timeout after publication started") + self.assertFalse(drain.wait(0.01)) + self.assertEqual(sample.features["aux"].shape, (8, 2)) + sample.release() + self.assertTrue(drain.wait(1)) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + with self.assertRaises(CaptureContractError): + _ = sample.features + + def test_hard_budget_oversize_and_passthrough(self): + for args in [(100, 0, 0), (8, 0, 500)]: + with self.assertRaises(AdmissionBudgetError): + self.controller.try_reserve_output(identity(), *args) + self.assertEqual(self.controller.stats()["output_reservations"], 0) + res = self.controller.try_reserve_output(identity(), 8, 0, 17) + self.assertEqual(res.host_bytes, 145) + self.controller.release_output(res) + self.controller.release_output(res) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_pinned_and_device_pressure_are_independent_hard_limits(self): + for limits in [(128, 1024, 1024), (1024, 1024, 128)]: + with self.subTest(limits=limits): + controller = CaptureController(self.cache, schema(), *limits) + reserved = controller.try_reserve_output(identity(30), 8, 0, 0) + self.assertIsNone(controller.try_reserve_output(identity(31), 4, 0, 0)) + self.assertEqual(controller.stats()["pinned_bytes"], 128) + self.assertEqual(controller.stats()["device_bytes"], 128) + with self.assertRaises(AdmissionBudgetError): + controller.try_reserve_output(identity(32), 9, 0, 0) + controller.release_output(reserved) + self.assertEqual(controller.stats()["output_bytes"], 0) + controller.shutdown() + + def test_admission_stats_count_attempt_reasons_and_separate_hard_oversize(self): + for kind, limits in (('pinned', (128, 1024, 1024)), + ('output', (1024, 128, 1024)), + ('device', (1024, 1024, 128))): + with self.subTest(kind=kind): + controller = CaptureController(self.cache, schema(), *limits) + try: + held = controller.try_reserve_output(identity(70), 8, 0, 0) + for _ in range(2): + self.assertIsNone(controller.try_reserve_output(identity(71), 4, 0, 0)) + with self.assertRaises(AdmissionBudgetError): + controller.try_reserve_output(identity(72), 100, 0, 0) + stats = controller.stats() + self.assertEqual(stats['admission_attempts'], 4) + self.assertEqual(stats['admission_reserved_attempts'], 1) + self.assertEqual(stats['admission_budget_declined_attempts'], 2) + self.assertEqual(stats[f'admission_declined_{kind}_attempts'], 2) + self.assertEqual(stats['admission_hard_oversize_attempts'], 1) + self.assertEqual(stats['admission_declined_controller_max_pending_attempts'], 0) + controller.release_output(held) + finally: + controller.shutdown() + + def test_controller_pending_and_fragmentation_have_distinct_admission_reasons(self): + self.controller._max_pending = 1 + held = self.controller.try_reserve_output(identity(80), 1, 0, 0) + self.assertIsNone(self.controller.try_reserve_output(identity(81), 1, 0, 0)) + self.controller.release_output(held) + self.controller.poll_drained() + self.assertEqual(self.controller.stats()['admission_declined_controller_max_pending_attempts'], 1) + self.controller._max_pending = 4096 + # Four 128-byte slots fill the 512-byte arena. Two separated free + # slots have enough total bytes for 192, but no contiguous DMA slot. + slots = [self.controller.try_reserve_output(identity(90 + i), 8, 0, 0) for i in range(4)] + self.controller.release_output(slots[0]) + self.controller.release_output(slots[2]) + self.controller.poll_drained() + self.assertIsNone(self.controller.try_reserve_output(identity(94), 12, 0, 0)) + stats = self.controller.stats() + self.assertEqual(stats['admission_declined_fragmentation_attempts'], 1) + self.assertEqual(stats['admission_declined_pinned_attempts'], 0) + self.assertEqual(stats['admission_declined_output_attempts'], 0) + self.assertEqual(stats['admission_declined_device_attempts'], 0) + for slot in slots: + self.controller.release_output(slot) + + def test_pinned_arena_is_one_retained_bounded_allocation_across_sizes(self): + real_empty = torch.empty + allocations = [] + def allocate(*args, **kwargs): + self.assertTrue(kwargs.pop("pin_memory")) + value = real_empty(*args, **kwargs) + allocations.append(value) + return value + with mock.patch("torch.empty", allocate): + for rows in (1, 8, 3, 16, 2): + views = self.controller._staging_views(rows) + for spec in schema().features: + value = views[spec.name] + self.assertEqual(tuple(value.shape), (rows, spec.width)) + self.assertEqual(value.untyped_storage().data_ptr(), + self.controller._pinned_arena.untyped_storage().data_ptr()) + self.assertEqual(len(allocations), 1) + self.assertEqual(allocations[0].numel(), 512) + self.assertEqual(self.controller.stats()["pinned_arena_bytes"], 512) + self.assertEqual(self.controller.stats()["pinned_bytes"], 0) + + def test_pinned_arena_capacity_is_conservative_and_alignment_is_charged(self): + controller = CaptureController(self.cache, schema(), 96, 1024, 1024) + self.assertEqual(controller.stats()["pinned_capacity_bytes"], 64) + with self.assertRaises(AdmissionBudgetError): + controller.try_reserve_output(identity(60), 5, 0, 0) + controller.shutdown() + mixed = FeatureSchema((FeatureSpec("aux", 1, "bfloat16", (1,)), + FeatureSpec("last_hidden", 1, "float32", ()))) + controller = CaptureController(PrefixFeatureCache(mixed, 1, 64), mixed, 16, 64, 64) + reservation = controller.try_reserve_output(identity(61), 1, 0, 0) + self.assertEqual(reservation.pinned_bytes, 8) + self.assertEqual(reservation.device_bytes, 6) + controller.release_output(reservation) + controller.shutdown() + + def test_quarantined_arena_forbids_new_reservation_and_landing_views(self): + self.controller._arena_poisoned = True + with self.assertRaisesRegex(CaptureContractError, "quarantined"): + self.controller.try_reserve_output(identity(62), 1, 0, 0) + with self.assertRaisesRegex(CaptureContractError, "quarantined"): + self.controller._staging_views(1) + + def test_admission_rollback_releases_frozen_lease_and_all_budgets(self): + populate(self.cache, tuple(range(8))) + plan = self.plan(L=4) + self.controller.release_output(plan.reservation) + self.cache.release_lease(plan.lease) + for key in ("output_bytes", "pinned_bytes", "device_bytes", "active_leases"): + self.assertEqual(self.controller.stats()[key], 0) + with self.assertRaises(CaptureContractError): + self.submit(plan) + + def test_cancel_waits_producer_event_and_future_cannot_cancel_cleanup(self): + plan = self.plan() + gate = GateEvent() + future = self.submit(plan, gate) + self.assertTrue(gate.entered.wait(3)) + self.assertFalse(future.cancel()) + drain = self.controller.invalidate(plan.execution, "retracted") + self.assertFalse(drain.done()) + self.assertEqual(self.controller.stats()["device_bytes"], 128) + gate.finished.set() + with self.assertRaises(CaptureCancelledError): + future.result(3) + self.assertTrue(drain.wait(1)) + for key in ("output_bytes", "pinned_bytes", "device_bytes", "active_leases", "pending_copies"): + self.assertEqual(self.controller.stats()[key], 0) + self.assertEqual(self.cache.stats()["ready_blocks"], 0) + + def test_old_epoch_completion_cannot_complete_new_execution(self): + old = self.plan(execution=identity(0, 0)) + gate = GateEvent() + old_future = self.submit(old, gate) + self.assertTrue(gate.entered.wait(3)) + self.controller.invalidate(old.execution, "retract") + new = self.plan(execution=identity(0, 1)) + new_future = self.submit(new) + gate.finished.set() + with self.assertRaises(CaptureCancelledError): + old_future.result(3) + self.take(new_future) + completions = self.controller.poll_completed() + self.assertEqual({c.execution for c in completions}, {old.execution, new.execution}) + self.assertIsInstance(next(c.error for c in completions if c.execution == old.execution), CaptureCancelledError) + self.assertIsNotNone(next(c.sample for c in completions if c.execution == new.execution)) + + def test_wrong_span_or_underlying_storage_is_rejected_before_submit(self): + plan = self.plan() + with self.assertRaises(CaptureContractError): + self.controller.submit(plan, CapturedSpan(plan.execution, 1, 8, features(7), None)) + too_large = features(8) + too_large["aux"] = torch.arange(1000, dtype=torch.float32)[:16].reshape(8, 2) + with self.assertRaises(CaptureContractError): + self.controller.submit(plan, CapturedSpan(plan.execution, 0, 8, too_large, None)) + self.assertEqual(self.controller.stats()["pending_copies"], 0) + self.controller.release_output(plan.reservation) + + def test_worker_failure_drains_and_future_reports_original_error(self): + plan = self.plan() + with mock.patch.object(self.controller, "_copy_suffix", side_effect=RuntimeError("D2H submission failed")): + future = self.submit(plan) + with self.assertRaisesRegex(RuntimeError, "D2H submission failed"): + future.result(3) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + self.assertEqual(self.controller.stats()["device_bytes"], 0) + self.assertEqual(self.cache.stats()["active_leases"], 0) + completions = self.controller.poll_completed() + self.assertEqual(len(completions), 1) + self.assertIsNotNone(completions[0].error) + + def test_cache_adoption_failure_keeps_complete_sample_delivery(self): + sample = self.take(self.submit(self.plan())) + self.assertTrue(torch.equal(sample.features["aux"], features(8)["aux"])) + with mock.patch.object(self.cache, "_adopt_released_sample", side_effect=RuntimeError("cache metadata failed")): + sample.release() + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.controller.stats()["fill_failures"], 1) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_cache_capacity_declines_adoption_without_blocking_release(self): + self.cache.capacity_bytes = 64 + sample = self.take(self.submit(self.plan())) + sample.release() + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.cache.stats()["cache_adoption_skips"], 1) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_release_transfers_same_storage_without_allocating_or_copying(self): + sample = self.take(self.submit(self.plan(n=9))) + pointers = {name: tensor.untyped_storage().data_ptr() for name, tensor in sample.features.items()} + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertEqual(self.controller.stats()["output_bytes"], 144) + original = self.cache._adopt_released_sample + def adopt(keys, values): + # The output charge remains live throughout cache admission. + self.assertEqual(self.controller.stats()["output_bytes"], 144) + return original(keys, values) + with mock.patch.object(self.cache, "_adopt_released_sample", adopt), \ + mock.patch("torch.empty", side_effect=AssertionError("release allocated a tensor")), \ + mock.patch.object(torch.Tensor, "copy_", side_effect=AssertionError("release copied a tensor")): + sample.release() + self.assertEqual(self.controller.stats()["output_bytes"], 0) + self.assertEqual(self.cache.stats()["cache_bytes"], 144) + self.assertEqual(self.cache.stats()["cache_logical_bytes"], 128) + self.assertEqual(self.cache.stats()["cache_owned_groups"], 1) + for entry in self.cache._entries.values(): + for name, tensor in entry.features.items(): + self.assertEqual(tensor.untyped_storage().data_ptr(), pointers[name]) + with self.assertRaises(CaptureContractError): + _ = sample.features + + def test_cancelled_sample_does_not_transfer_output_to_cache(self): + plan = self.plan() + sample = self.take(self.submit(plan)) + drain = self.controller.invalidate(plan.execution, "retract after assembly") + self.assertFalse(drain.done()) + sample.release() + self.assertTrue(drain.wait(1)) + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + + def test_no_complete_chunk_does_not_retain_full_output(self): + sample = self.take(self.submit(self.plan(n=3))) + references = [weakref.ref(tensor) for tensor in sample.features.values()] + sample.release() + gc.collect() + self.assertEqual(self.cache.stats()["cache_bytes"], 0) + self.assertTrue(all(reference() is None for reference in references)) + + def test_workers_assemble_independent_jobs_concurrently(self): + gate = GateEvent() + first = self.submit(self.plan(execution=identity(60)), gate) + self.assertTrue(gate.entered.wait(3)) + second = self.submit(self.plan(execution=identity(61))) + second_sample = self.take(second) + self.assertFalse(first.done()) + self.assertTrue(torch.equal(second_sample.features["aux"], features(8)["aux"])) + gate.finished.set() + self.take(first) + + def test_reserved_slots_are_disjoint_and_coalesce_only_after_release(self): + first = self.controller.try_reserve_output(identity(70), 8, 0, 0) + second = self.controller.try_reserve_output(identity(71), 8, 0, 0) + real_empty = torch.empty + def allocate(*args, **kwargs): + kwargs.pop("pin_memory", None) + return real_empty(*args, **kwargs) + with mock.patch("torch.empty", allocate): + a = self.controller._staging_views(8, first) + b = self.controller._staging_views(8, second) + for value in a.values(): + value.fill_(10) + for value in b.values(): + value.fill_(20) + self.assertTrue(all(bool(torch.all(value == 10)) for value in a.values())) + self.assertTrue(all(bool(torch.all(value == 20)) for value in b.values())) + # No reader remains when a slot is explicitly released here. + a.clear() + self.controller.release_output(first) + with self.assertRaises(CaptureContractError): + self.controller._staging_views(8, first) + third = self.controller.try_reserve_output(identity(72), 8, 0, 0) + c = self.controller._staging_views(8, third) + for value in c.values(): + value.fill_(30) + self.assertTrue(all(bool(torch.all(value == 20)) for value in b.values())) + b.clear() + c.clear() + self.controller.release_output(second) + self.controller.release_output(third) + self.assertEqual(self.controller._pinned_free, [(0, 512)]) + + def test_corrupt_lease_fails_instead_of_zero_filling(self): + populate(self.cache, tuple(range(8))) + plan = self.plan(L=4) + entry = self.cache._entries[self.cache.block_keys("ns", tuple(range(8)))[0]] + entry.features.pop("last_hidden") + future = self.submit(plan) + with self.assertRaisesRegex(CaptureContractError, "missing or corrupt"): + future.result(3) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_shutdown_nonblocking_keeps_inflight_storage_until_actual_drain(self): + plan = self.plan() + gate = GateEvent() + future = self.submit(plan, gate) + self.assertTrue(gate.entered.wait(3)) + self.controller.shutdown(wait=False) + self.assertEqual(self.controller.stats()["device_bytes"], 128) + gate.finished.set() + with self.assertRaises(CaptureCancelledError): + future.result(3) + self.controller.shutdown(wait=True) + self.assertEqual(self.controller.stats()["output_bytes"], 0) + + def test_partial_copy_and_failed_wait_drain_both_owners_without_traceback_storage(self): + # CPU tensors plus fully mocked CUDA entry points test scheduling/lifetime + # logic deterministically. This is not CUDA numerical/device evidence. + real_copy, real_empty = torch.Tensor.copy_, torch.empty + for fail_wait in (False, True): + with self.subTest(fail_wait=fail_wait): + plan = self.plan(execution=identity(20 + int(fail_wait))) + allocated, retained_errors = [], [] + copies = [0] + class Producer: + syncs = 0 + def synchronize(producer): + producer.syncs += 1 + producer = Producer() + class Stream: + syncs = 0 + def wait_event(stream, event): + if fail_wait: + raise RuntimeError("wait_event failed before dependency") + def synchronize(stream): + stream.syncs += 1 + stream = Stream() + def allocate(*args, **kwargs): + kwargs.pop("pin_memory", None) + tensor = real_empty(*args, **kwargs) + allocated.append(weakref.ref(tensor)) + return tensor + def copy(tensor, *args, **kwargs): + copies[0] += 1 + if copies[0] == 2: + raise RuntimeError(tensor) + return real_copy(tensor, *args, **kwargs) + original = self.controller._copy_suffix + def pretend_cuda(job): + job.is_cuda = True + return original(job) + with mock.patch.object(self.controller, "_copy_suffix", pretend_cuda), \ + mock.patch("torch.cuda.device", return_value=nullcontext()), \ + mock.patch("torch.cuda.stream", return_value=nullcontext()), \ + mock.patch("torch.cuda.Stream", return_value=stream), \ + mock.patch("torch.empty", allocate), \ + mock.patch.object(torch.Tensor, "record_stream", return_value=None), \ + mock.patch.object(torch.Tensor, "copy_", copy): + future = self.submit(plan, producer) + try: + future.result(3) + except RuntimeError as error: + retained_errors.append(error) + gc.collect() + self.assertEqual(len(retained_errors), 1) + self.assertEqual(producer.syncs, 1) + self.assertEqual(stream.syncs, 1) + # The single explicitly accounted arena intentionally survives + # safe failures; no per-copy views/source tensors may survive. + self.assertTrue(all(ref() is None or ref() is self.controller._pinned_arena + for ref in allocated)) + expected_arena = (self.controller._pinned_arena.numel() + if self.controller._pinned_arena is not None else 0) + self.assertEqual(self.controller.stats()["pinned_arena_bytes"], expected_arena) + self.assertLessEqual(expected_arena, self.controller.stats()["pinned_capacity_bytes"]) + for key in ("output_bytes", "pinned_bytes", "device_bytes", "pending_copies"): + self.assertEqual(self.controller.stats()[key], 0) + self.controller.poll_completed() + self.controller.poll_drained() + + def test_unproven_event_drain_quarantines_without_budget_reuse(self): + cache = PrefixFeatureCache(schema(), 4, 128) + controller = CaptureController(cache, schema(), 128, 128, 128) + execution = identity(99) + tokens = tuple(range(8)) + lease = cache.lookup_and_pin("ns", tokens, 8) + reservation = controller.try_reserve_output(execution, 8, 0, 0) + plan = controller.freeze_plan(execution, "ns", tokens, 8, 0, lease, reservation) + class UnprovenEvent: + def synchronize(self): + raise RuntimeError("producer drain unavailable") + future = controller.submit(plan, CapturedSpan(execution, 0, 8, features(8), UnprovenEvent())) + with self.assertRaisesRegex(RuntimeError, "quarantined"): + future.result(3) + drain = controller.invalidate(execution, "cancel after failed event") + self.assertFalse(drain.wait(0.01)) + controller.release_output(reservation) + self.assertEqual(controller.stats()["device_bytes"], 128) + self.assertEqual(controller.stats()["quarantined_executions"], 1) + with self.assertRaisesRegex(RuntimeError, "not drained"): + controller.shutdown(wait=True) + + def test_completion_requires_exactly_one_outcome(self): + with self.assertRaises(CaptureContractError): + CaptureCompletion(identity()) + with self.assertRaises(CaptureContractError): + CaptureCompletion(identity(), sample=object(), error=RuntimeError()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_topology.py b/tests/test_runtime/test_hs_topology.py new file mode 100644 index 0000000000..e57928ba72 --- /dev/null +++ b/tests/test_runtime/test_hs_topology.py @@ -0,0 +1,90 @@ +"""CPU startup topology contracts using the real SG ParallelContext getter.""" + +import os +from pathlib import Path +import sys +from types import SimpleNamespace +import unittest +from unittest import mock + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "plugins/sglang-spec-capture")) + +from sglang.srt import runtime_context +from sglang.srt.arg_groups.model_override_base import resolved_view +from sglang.srt.managers import prefix_admission +from specforge_sglang_capture.hs.config import HSConfig +from specforge_sglang_capture.hs.protocol import CaptureContractError +from specforge_sglang_capture.hs.runtime import HSCaptureRuntime + + +class HSStartupTopologyTests(unittest.TestCase): + def setUp(self): + self.args = SimpleNamespace( + enable_dp_attention=False, enable_hierarchical_cache=False, + enable_lora=False, enable_session_radix_cache=False, + disable_radix_cache=False, enable_memory_saver=False, + enable_deterministic_inference=True, model_path="/cpu-fixture", + revision=None, quantization=None, load_format="auto", kv_cache_dtype="auto", + bf16_gemm_backend="auto", attention_backend="flashinfer", prefill_attention_backend=None) + text = SimpleNamespace(hidden_size=4, to_dict=lambda: {"hidden_size": 4}) + runtime = HSCaptureRuntime.__new__(HSCaptureRuntime) + runtime.s = SimpleNamespace(server_args=self.args, enable_priority_preemption=False, + prefill_delayer=None, dllm_config=None, spec_algorithm=SimpleNamespace(is_none=lambda: True), + page_size=1, model_config=SimpleNamespace(hf_text_config=text, dtype=torch.float32)) + runtime.config, runtime.enabled = HSConfig(capture=True, cache=True, chunk_tokens=4), True + self.runtime = runtime + self.parallel = runtime_context.get_parallel() + self.assertIsInstance(self.parallel, runtime_context.ParallelContext) + self.env = mock.patch.dict(os.environ, {"MOONCAKE_PROTOCOL": "tcp", "SGLANG_SPEC_CAPTURE_GPU_PUT": "0"}) + self.env.start() + self.addCleanup(self.env.stop) + + def identity(self, **overrides): + topology = dict(tp_size=2, tp_rank=0, pp_size=1, pp_rank=0, dp_size=1, + attn_dp_size=1, attn_dp_rank=0, attn_cp_size=1, attn_cp_rank=0, + attn_tp_size=2, attn_tp_rank=0, dcp_size=1) + topology.update(overrides) + features = SimpleNamespace(return_aux_hidden_state_layer_ids=[1, 3], aux_hidden_state_capture="dflash") + with self.parallel.override(**topology), \ + mock.patch.object(runtime_context, "get_schedule", return_value=SimpleNamespace(chunked_prefill_size=-1)), \ + mock.patch.object(runtime_context, "get_exec", return_value=SimpleNamespace(features=features)), \ + mock.patch.object(prefix_admission, "validate_capture_checkpoint_support", + return_value={"backend": "isolated-cpu-fixture"}) as checkpoint: + # get_parallel itself is real. The unrelated numerical-backend gate + # is isolated; unsupported topology must fail before invoking it. + try: + return self.runtime._model_identity(), checkpoint.call_count + finally: + self.checkpoint_calls = checkpoint.call_count + + def test_attention_dp_is_rejected_with_false_deprecated_flag(self): + self.assertIs(resolved_view(self.args).enable_dp_attention, False) + with self.assertRaisesRegex(CaptureContractError, "attention_DP"): + self.identity(attn_dp_size=2, attn_tp_size=1) + self.assertEqual(self.checkpoint_calls, 0) + self.assertFalse(torch.cuda.is_initialized()) + + def test_ordinary_tp1_and_tp2_attention_dp1_keep_startup_identity(self): + for width in (1, 2): + with self.subTest(tp_size=width): + (manifest, schema), checkpoint_calls = self.identity(tp_size=width, attn_tp_size=width) + self.assertEqual(manifest["tp_size"], width) + self.assertEqual([feature.name for feature in schema.features], ["aux", "last_hidden"]) + self.assertEqual(checkpoint_calls, 1) + self.assertFalse(torch.cuda.is_initialized()) + + def test_replica_dp_pp_and_cp_startup_rejections_remain(self): + for topology, diagnostic in ((dict(dp_size=2), "DP"), (dict(pp_size=2), "PP"), + (dict(attn_cp_size=2, attn_tp_size=1), "CP"), + (dict(dcp_size=2), "CP")): + with self.subTest(topology=topology): + with self.assertRaisesRegex(CaptureContractError, diagnostic): + self.identity(**topology) + self.assertEqual(self.checkpoint_calls, 0) + self.assertFalse(torch.cuda.is_initialized()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_tp_protocol.py b/tests/test_runtime/test_hs_tp_protocol.py new file mode 100644 index 0000000000..fa0ba7f791 --- /dev/null +++ b/tests/test_runtime/test_hs_tp_protocol.py @@ -0,0 +1,478 @@ +"""Threaded CPU contracts for actual HS callbacks and SG's outer phase. + +The rendezvous enforces collective operation order and exchanges real Python +payloads between two scheduler-owner threads. It is not CUDA, IPC or actual TP +execution. Point SPECFORGE_HS_TEST_SG_SOURCE at the companion SG checkout when +the interpreter's editable still points at the official baseline. +""" + +import copy +import dataclasses +import importlib.util +import json +import os +from pathlib import Path +import sys +import threading +import time +from types import SimpleNamespace +import unittest +from unittest import mock + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "plugins/sglang-spec-capture")) + +from specforge_sglang_capture.capture import CaptureRuntime +from specforge_sglang_capture.hs.attempts import digest_request +from specforge_sglang_capture.hs.config import HSConfig +from specforge_sglang_capture.hs.protocol import FeatureSchema, FeatureSpec +from specforge_sglang_capture.hs.runtime import HSCaptureRuntime + + +def _sg_module(): + if source := os.environ.get("SPECFORGE_HS_TEST_SG_SOURCE"): + path = Path(source) / "python/sglang/srt/managers/prefix_admission.py" + else: + spec = importlib.util.find_spec("sglang") + if spec is None or spec.origin is None: + return None + path = Path(spec.origin).parent / "srt/managers/prefix_admission.py" + if not path.is_file(): + return None + spec = importlib.util.spec_from_file_location("_specforge_hs_test_prefix_admission", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +SG = _sg_module() + + +class _Rendezvous: + def __init__(self): + self.condition = threading.Condition() + self.steps, self.trace = {}, [[], []] + self.failed = None + + def call(self, rank, step, operation, value): + with self.condition: + if self.failed: + raise RuntimeError(self.failed) + self.trace[rank].append((operation, value[0] if operation == "gather" + and isinstance(value, tuple) and len(value) == 2 else None)) + slot = self.steps.setdefault(step, {"operation": operation, "values": {}}) + if slot["operation"] != operation or rank in slot["values"]: + self.failed = f"collective order mismatch at step {step}: {slot['operation']} / {operation}" + self.condition.notify_all() + raise RuntimeError(self.failed) + slot["values"][rank] = value + if len(slot["values"]) == 2: + slot["result"] = ([slot["values"][r] for r in (0, 1)] + if operation == "gather" else slot["values"][0]) + self.condition.notify_all() + deadline = time.monotonic() + 5 + while "result" not in slot and not self.failed: + remaining = deadline - time.monotonic() + if remaining <= 0: + self.failed = f"rank {rank} timed out at collective {step} ({operation})" + self.condition.notify_all() + break + self.condition.wait(remaining) + if self.failed: + raise RuntimeError(self.failed) + return copy.deepcopy(slot["result"]) + + +class _Sink: + aux_layer_ids = [1, 3] + + def close(self): + pass + + +def _raise(message): + raise RuntimeError(message) + + +def _schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1, 3)), + FeatureSpec("last_hidden", 2, "float32", ()))) + + +@unittest.skipIf(SG is None, "companion native SG prefix-admission module is unavailable") +class ThreadedTPProtocolTests(unittest.TestCase): + def pair(self, operation, *, initialize=None, configure=None): + rendezvous = _Rendezvous() + results, errors, snapshots = [None, None], [None, None], [None, None] + + def run(rank): + step = 0 + + def collective(operation, value): + nonlocal step + current, step = step, step + 1 + return rendezvous.call(rank, current, operation, value) + + scheduler = SimpleNamespace( + prefix_admission_broadcast=lambda value: collective("broadcast", value), + prefix_admission_gather=lambda value: collective("gather", value), + abort_request=lambda req: None) + coordinator = SG.PrefixAdmissionCoordinator.__new__(SG.PrefixAdmissionCoordinator) + coordinator.gather = scheduler.prefix_admission_gather + coordinator.broadcast = scheduler.prefix_admission_broadcast + + def phase(call): + value, failures = coordinator.phase(call) + if any(error is not None for error in failures): + return None, failures + return coordinator.broadcast(value), failures + + runtime = None + try: + native = CaptureRuntime(sink=_Sink(), is_writer=rank == 0, + gpu_put=False, max_pending_batches=2, defer_response=lambda *args: None) + config = HSConfig(capture=True, cache=True, chunk_tokens=4, + cache_bytes=256, pinned_bytes=256, output_bytes=512, + device_bytes=256, attempt_capacity=2) + kwargs = {"manifest": {"model": "threaded-cpu"}, "schema": _schema()} + if initialize is not None: + config, kwargs = initialize(rank, config, kwargs) + runtime = HSCaptureRuntime(scheduler, native, config, tp_size=2, tp_rank=rank, **kwargs) + if configure is not None: + configure(rank, runtime) + results[rank] = operation(rank, runtime, phase) + except BaseException as exc: + errors[rank] = exc + finally: + if runtime is not None: + snapshots[rank] = {"samples": set(runtime.registry.active), + "tokens": set(runtime.registry.tokens)} + try: + # Cleanup is local even when a protocol failure has made + # further collectives forbidden. Do not call control. + if runtime.controller is not None: + runtime.controller.shutdown(wait=True) + except Exception as exc: + if errors[rank] is None: + errors[rank] = exc + + threads = [threading.Thread(target=run, args=(rank,), daemon=True) for rank in (0, 1)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(15) + self.assertFalse(any(thread.is_alive() for thread in threads), "scheduler CPU contract thread hung") + self.assertIsNone(rendezvous.failed) + self.assertEqual([item[0] for item in rendezvous.trace[0]], + [item[0] for item in rendezvous.trace[1]]) + self.assertEqual(len(rendezvous.trace[0]), len(rendezvous.trace[1])) + self.assertFalse(torch.cuda.is_initialized()) + return results, errors, snapshots, rendezvous.trace + + def body(self, runtime, action="begin", sample="held", **values): + return {"protocol_version": 1, "server_epoch": runtime.epoch, "action": action, + "store_id": "store", "sample_id": sample, "gen": 1, + "attempt_id": "attempt-" + sample, "request_digest": "digest", + "publication_ownership": "attempt-v1", **values} + + def begin(self, runtime, phase): + reply, failures = phase(lambda: runtime.control(runtime.s, self.body(runtime))) + self.assertFalse(any(failures)) + return reply, runtime.registry.tokens[reply["reservation_token"]] + + def failed(self, result): + values, errors, _, _ = result + self.assertEqual(errors, [None, None]) + for value, failures in values: + self.assertIsNone(value) + self.assertTrue(all(isinstance(error, str) for error in failures)) + + def test_control_poll_owner_and_writer_payload_failures_vote_before_broadcast(self): + for fault in ("owner", "poll", "transfer-payload", "capability"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + if fault == "owner" and rank == 1: + runtime._assert_owner = lambda: _raise("peer owner failure") + elif fault == "poll" and rank == 0: + runtime.controller.poll_completed = lambda: _raise("writer poll failure") + elif fault == "transfer-payload" and rank == 0: + self.fail_build(runtime, "control.transfers.payload") + elif fault == "capability" and rank == 0: + runtime.capability = lambda: _raise("writer capability failure") + return phase(lambda: runtime.control(runtime.s, + {"protocol_version": 1, "action": "capability"})) + self.failed(self.pair(operation)) + + @staticmethod + def fail_build(runtime, target): + original = runtime._writer_broadcast + + def broadcast(phase, build): + return original(phase, lambda: _raise("writer payload failure") if phase == target else build()) + runtime._writer_broadcast = broadcast + + def test_transfer_peer_retirement_failure_votes_before_next_control_or_prepare_collective(self): + for action in ("control", "prepare"): + with self.subTest(action=action): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + if rank == 0: + entry.state = "PUBLISHED" + runtime.registry.retire(entry) + runtime._transferred_tokens.append(entry.token) + else: + runtime.registry.retire = lambda entry: _raise("peer transfer apply failed") + if action == "control": + return phase(lambda: runtime.control(runtime.s, + {"protocol_version": 1, "action": "capability"})) + req = SimpleNamespace(rid="r", sampling_params=SimpleNamespace( + custom_params={"spec_capture": {"protocol_version": 1}})) + return phase(lambda: runtime.prepare(runtime.s, req)) + self.failed(self.pair(operation)) + + def test_control_lookup_and_active_retired_branch_disagreement_fail_together(self): + for fault in ("missing", "active-retired", "request-action"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + if rank == 1: + if fault == "missing": + runtime.registry.tokens.pop(entry.token) + elif fault == "active-retired": + entry.state = "PUBLISHED" + runtime.registry.retire(entry) + body = {**self.body(runtime, "resolve"), "reservation_token": reply["reservation_token"]} + if rank == 1 and fault == "request-action": + body["action"] = "capability" + return phase(lambda: runtime.control(runtime.s, body)) + self.failed(self.pair(operation)) + + def test_resolve_invalidate_abort_and_wire_errors_vote_before_later_broadcast(self): + for fault in ("invalidate", "abort", "wire"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + req = SimpleNamespace(rid="request", _specforge_hs_attempt=entry) + entry.request = req + if fault == "invalidate" and rank == 0: + runtime.invalidate = lambda *args: _raise("writer invalidation failed") + elif fault == "abort" and rank == 1: + runtime.s.abort_request = lambda req: _raise("peer abort failed") + elif fault == "wire" and rank == 0: + runtime._wire = lambda entry: _raise("writer wire failed") + body = {**self.body(runtime, "resolve"), + "reservation_token": reply["reservation_token"], "cancel": fault != "wire"} + return phase(lambda: runtime.control(runtime.s, body)) + with mock.patch.dict(sys.modules, {"sglang.srt.managers.io_struct": + SimpleNamespace(AbortReq=lambda **kwargs: SimpleNamespace(**kwargs))}): + self.failed(self.pair(operation)) + + def test_begin_preflight_never_emits_false_no_grant_on_divergence_or_partial_commit(self): + for fault in ("tokens", "preflight", "reply", "commit", "capacity-disagreement"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + if fault == "capacity-disagreement": + self.begin(runtime, phase) + self.control_success(runtime, phase, "other") + if rank == 0: + runtime.registry.active.clear() + runtime.registry.tokens.clear() + if fault == "tokens" and rank == 0: + self.fail_build(runtime, "control.begin.tokens") + elif fault == "preflight" and rank == 1: + runtime._begin_trial = lambda *args: _raise("peer preflight failed") + elif fault == "reply" and rank == 0: + runtime._wire = lambda entry: _raise("writer reply failed") + elif fault == "commit" and rank == 1: + original = runtime._local_phase + def local(phase, call): + return original(phase, lambda: _raise("peer commit failed") + if phase == "control.begin.commit" else call()) + runtime._local_phase = local + return phase(lambda: runtime.control(runtime.s, self.body(runtime, sample="new"))) + result = self.pair(operation) + self.failed(result) + snapshots = result[2] + new = ("store", "new", 1) + if fault == "commit": + self.assertIn(new, snapshots[0]["samples"]) + self.assertNotIn(new, snapshots[1]["samples"]) + else: + self.assertTrue(all(new not in snapshot["samples"] for snapshot in snapshots)) + + def control_success(self, runtime, phase, sample): + reply, failures = phase(lambda: runtime.control(runtime.s, self.body(runtime, sample=sample))) + self.assertFalse(any(failures)) + return reply + + def test_unanimous_capacity_decline_is_atomic_and_typed(self): + def operation(rank, runtime, phase): + self.begin(runtime, phase) + self.control_success(runtime, phase, "other") + return phase(lambda: runtime.control(runtime.s, self.body(runtime, sample="new"))) + values, errors, snapshots, _ = self.pair(operation) + self.assertEqual(errors, [None, None]) + for reply, failures in values: + self.assertFalse(any(failures)) + self.assertEqual(reply["error_code"], "capture_capacity_declined") + self.assertIs(reply["reservation_granted"], False) + self.assertTrue(all(len(snapshot["samples"]) == 2 for snapshot in snapshots)) + + def test_read_cleanup_validation_and_existing_permit_branches_are_voted(self): + for fault in ("read-validate", "read-branch", "read-token", "cleanup-validate", + "cleanup-branch", "cleanup-token", "cleanup-done", "cleanup-retire", "retired-read"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + body = {**self.body(runtime), "reservation_token": reply["reservation_token"]} + if fault.startswith("read"): + entry.state = "PUBLISHED" + body.update(action="read_acquire", reader_owner="owner") + if fault == "read-validate" and rank == 1: + entry.released_readers.append(("owner", "released-token")) + elif fault == "read-branch" and rank == 1: + entry.readers["owner"] = "existing-token" + elif fault == "read-token" and rank == 0: + self.fail_build(runtime, "control.read-acquire.token") + elif fault == "retired-read": + entry.state = "PUBLISHED" + runtime.registry.retire(entry) + if rank == 0: + entry.released_readers.append(("owner", "released-token")) + body.update(action="read_release", reader_owner="owner", reader_token="released-token") + else: + entry.revoked, entry.state = True, "WRITER_DRAINED" + body.update(action="cleanup_acquire", cleanup_owner="owner") + if fault == "cleanup-validate" and rank == 1: + entry.readers["reader"] = "reader-token" + elif fault == "cleanup-branch" and rank == 1: + entry.cleanup_owner, entry.cleanup_token = "owner", "existing-token" + elif fault == "cleanup-token" and rank == 0: + self.fail_build(runtime, "control.cleanup-acquire.token") + elif fault in ("cleanup-done", "cleanup-retire"): + entry.cleanup_owner, entry.cleanup_token = "owner", "cleanup-token" + body.update(action="cleanup_done", cleanup_token="cleanup-token") + if fault == "cleanup-done" and rank == 1: + entry.cleanup_token = "different-token" + elif fault == "cleanup-retire" and rank == 1: + runtime.registry.retire = lambda entry: _raise("peer retirement failed") + return phase(lambda: runtime.control(runtime.s, body)) + self.failed(self.pair(operation)) + + def test_prepare_agrees_membership_and_votes_local_parse_and_lease_failures(self): + for fault in ("membership", "parse", "lease"): + with self.subTest(fault=fault): + def operation(rank, runtime, phase): + tokens = list(range(8)) + spec = {"features": {"aux": "aux", "last_hidden": "last"}, "passthrough": []} + body = self.body(runtime, request_digest=digest_request(tokens, spec)) + reply, failures = phase(lambda: runtime.control(runtime.s, body)) + self.assertFalse(any(failures)) + spec.update({key: reply[key] for key in ( + "server_epoch", "attempt_id", "reservation_token", "object_key_layout", + "store_id", "sample_id", "gen", "request_digest")}) + spec.update(protocol_version=1, capture_namespace=runtime.namespace, + capture_signature=runtime.signature) + req = SimpleNamespace(rid="request", origin_input_ids=tokens, + sampling_params=SimpleNamespace(custom_params={"spec_capture": spec}, max_new_tokens=0)) + if fault == "membership" and rank == 1: + req.sampling_params.custom_params = None + elif fault == "parse" and rank == 1: + req.sampling_params = None + elif fault == "lease" and rank == 0: + runtime.cache.lookup_and_pin = lambda *args: _raise("writer lease lookup failed") + outcome = phase(lambda: runtime.prepare(runtime.s, req)) + runtime.rollback(runtime.s, req, None, "CPU contract teardown") + return outcome + self.failed(self.pair(operation)) + + def test_startup_epoch_model_identity_and_core_failures_terminate_both_ranks(self): + for fault in ("epoch", "model", "identity", "core"): + with self.subTest(fault=fault): + original = HSCaptureRuntime._writer_broadcast + def broadcast(runtime, phase, build): + return original(runtime, phase, lambda: _raise("writer epoch failure") + if fault == "epoch" and phase == "startup.epoch" else build()) + def initialize(rank, config, kwargs): + if fault == "model": + kwargs = {"manifest": None, "schema": None} + elif fault == "identity" and rank == 1: + kwargs["schema"] = None + # Keep the identity-build failure after model resolution. + kwargs["schema"] = SimpleNamespace(features=(object(),)) + elif fault == "core" and rank == 0: + config = dataclasses.replace(config, max_workers=0) + return config, kwargs + def model(runtime): + if not runtime.writer: + _raise("peer backend capability failure") + return {"model": "threaded-cpu"}, _schema() + with mock.patch.object(HSCaptureRuntime, "_writer_broadcast", broadcast), \ + mock.patch.object(HSCaptureRuntime, "_model_identity", model): + values, errors, _, _ = self.pair( + lambda rank, runtime, phase: self.fail("failed startup returned a runtime"), + initialize=initialize) + self.assertEqual(values, [None, None]) + self.assertTrue(all(error is not None for error in errors)) + + def test_successful_permit_lifecycle_and_native_prepare_keep_phase_order(self): + def operation(rank, runtime, phase): + reply, entry = self.begin(runtime, phase) + wire = {**self.body(runtime), "reservation_token": reply["reservation_token"]} + entry.state = "PUBLISHED" + read, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "read_acquire", "reader_owner": "reader"})) + self.assertFalse(any(failures)) + released, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "read_release", "reader_owner": "reader", "reader_token": read["reader_token"]})) + self.assertFalse(any(failures)) + entry.revoked, entry.state = True, "WRITER_DRAINED" + cleanup, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "cleanup_acquire", "cleanup_owner": "cleanup"})) + self.assertFalse(any(failures)) + retired, failures = phase(lambda: runtime.control(runtime.s, + {**wire, "action": "cleanup_done", "cleanup_token": cleanup["cleanup_token"]})) + self.assertFalse(any(failures)) + req = SimpleNamespace(rid="native", sampling_params=SimpleNamespace(custom_params=None)) + value, failures = phase(lambda: runtime.prepare(runtime.s, req)) + self.assertFalse(any(failures)) + self.assertIsNone(value) + return retired + values, errors, snapshots, _ = self.pair(operation) + self.assertEqual(errors, [None, None]) + self.assertTrue(all(value["state"] == "RETIRED" for value in values)) + self.assertTrue(all(not snapshot["tokens"] for snapshot in snapshots)) + + def test_committed_diagnostics_keep_each_tp_rank_without_scheduler_rank_attribute(self): + def operation(rank, runtime, phase): + tokens = list(range(8)) + spec = {"features": {"aux": "aux", "last_hidden": "last"}, "passthrough": []} + body = self.body(runtime, request_digest=digest_request(tokens, spec)) + reply, failures = phase(lambda: runtime.control(runtime.s, body)) + self.assertFalse(any(failures)) + spec.update({key: reply[key] for key in ( + "server_epoch", "attempt_id", "reservation_token", "object_key_layout", + "store_id", "sample_id", "gen", "request_digest")}) + spec.update(protocol_version=1, capture_namespace=runtime.namespace, + capture_signature=runtime.signature) + req = SimpleNamespace(rid="request", origin_input_ids=tokens, + sampling_params=SimpleNamespace(custom_params={"spec_capture": spec}, max_new_tokens=0)) + proposal, failures = phase(lambda: runtime.prepare(runtime.s, req)) + self.assertFalse(any(failures)) + # The helper broadcasts the writer's proposal. Each rank's opaque + # candidate remains local, as it does in actual SG coordination. + phase(lambda: runtime.after_match(runtime.s, req, proposal, 0)) + phase(lambda: runtime.reserve(runtime.s, req, proposal, 0)) + phase(lambda: runtime.commit(runtime.s, req, proposal, 0)) + self.assertFalse(hasattr(runtime.s, "tp_rank")) + return req._specforge_hs_attempt.last_plan["rank"] + + values, errors, _, _ = self.pair(operation) + self.assertEqual(errors, [None, None]) + self.assertEqual(values, [0, 1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runtime/test_hs_unused_work.py b/tests/test_runtime/test_hs_unused_work.py new file mode 100644 index 0000000000..e86bf0afe2 --- /dev/null +++ b/tests/test_runtime/test_hs_unused_work.py @@ -0,0 +1,162 @@ +"""No-op cache work must disappear without weakening content or ownership checks.""" + +import hashlib +import struct +import unittest +from dataclasses import replace +from unittest import mock + +import torch + +from specforge_sglang_capture.hs import ( + CaptureContractError, CaptureController, CapturedSpan, ExecutionIdentity, + FeatureSchema, FeatureSpec, PrefixFeatureCache, +) + + +def schema(): + return FeatureSchema((FeatureSpec("aux", 2, "float32", (1,)),)) + + +class UnusedCacheWorkTests(unittest.TestCase): + def cache(self, capacity=1024, enabled=True): + with mock.patch.dict("os.environ", {"SPECFORGE_HS_SKIP_UNUSED_CACHE_WORK": str(int(enabled))}): + return PrefixFeatureCache(schema(), 4, capacity) + + def test_empty_lease_skips_hash_but_validates_all_tokens_and_identity(self): + cache = self.cache() + tokens = tuple(range(8)) + lease = cache.lookup_and_pin("ns", tokens, 0) + with mock.patch.object(cache, "block_keys", side_effect=AssertionError("unnecessary hash")): + cache._validate_plan_lease(lease, 0, "ns", tokens) + for bad in [(True,), (-1,), (2**63,), [0]]: + with self.assertRaises(CaptureContractError): + cache._validate_plan_lease(lease, 0, "ns", bad) + for handle, end, namespace in [(replace(lease), 0, "ns"), (lease, 1, "ns"), + (lease, 0, "other"), (lease, -1, "ns")]: + with self.assertRaises(CaptureContractError): + cache._validate_plan_lease(handle, end, namespace, tokens) + cache.release_lease(lease) + with self.assertRaises(CaptureContractError): + cache._validate_plan_lease(lease, 0, "ns", tokens) + + def test_disabled_cache_lookup_validates_namespace_and_tokens_without_content_hash(self): + cache = self.cache(0) + with mock.patch.object(cache, "block_keys", side_effect=AssertionError("unnecessary hash")): + lease = cache.lookup_and_pin("ns", tuple(range(8)), 8) + self.assertEqual(lease.coverage_end, 0) + self.assertEqual(cache.stats()["active_leases"], 1) + cache.release_lease(lease) + for namespace, tokens, limit in [("", (0,), 1), (None, (0,), 1), + ("ns", (True,), 1), ("ns", (0,), -1)]: + with self.assertRaises(CaptureContractError): + cache.lookup_and_pin(namespace, tokens, limit) + + def test_protocol_bytes_and_parent_chain_match_independent_v1_oracle(self): + tokens = (0, 1, 2**31, 2**63-1, 9, 8, 7, 6, 5) + namespace = "model-测试" + encoded = namespace.encode() + parent = hashlib.sha256(b"spec-prefix-root-v1\0" + struct.pack(" capacity: + raise AssertionError("registration exceeded endpoint bound") + return server(url, json_body, timeout) + adapter = _adapter(store, post) + adapter.hs_capture = adapter.hs_prefix_cache = True + from specforge.inference.capture_protocol import CaptureProtocol + adapter._capture_protocol = CaptureProtocol(adapter.base_url, post, 1) + channel = StreamingRefChannel(os.path.join(root, "refs")) + workers, drive = _build([adapter], _prompts(4), store, channel, lease=lease, producer_concurrency=concurrency, max_worker_failures=1, max_prompt_attempts=1, backpressure_poll_s=.001, peer_wait_timeout_s=2) + self.assertEqual(drive(), 4) + self.assertEqual(workers[0].controller.status()["prompts_failed"], 0) + self.assertEqual(channel.published, 4) + self.assertTrue(sizes) + self.assertLessEqual(max(sizes), capacity) + self.assertFalse(store._capture_pending_begin) + + def test_refused_and_new_prepared_groups_can_share_the_next_begin(self): + from specforge.inference.capture_protocol import CaptureCapacityDeclined + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + refuse = [True] + def post(url, json_body, timeout): + if json_body.get("action") == "begin_batch" and refuse[0]: + refuse[0] = False + return dict(error="full", error_code="capture_capacity_declined", reservation_granted=False) + return server(url, json_body, timeout) + adapter = SGLangServerCaptureAdapter("http://server", store, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), post_fn=post, control_fn=post, hs_capture=True, hs_prefix_cache=True) + old = adapter.prepare_refs_batches([[_task(0, 5)]], capture=_eagle3_contract()) + with self.assertRaises(CaptureCapacityDeclined): + adapter.register_prepared_batches(old) + attempt = old[0]["payloads"][0]["attempt_id"] + fresh = adapter.prepare_refs_batches([[_task(1, 5)]], capture=_eagle3_contract()) + adapter.register_prepared_batches(old + fresh) + self.assertEqual(old[0]["payloads"][0]["attempt_id"], attempt) + self.assertFalse(store._capture_pending_begin) + for group in old + fresh: + self.assertEqual(len(adapter.produce_prepared(group, capture=_eagle3_contract())), 1) + + def test_atomic_refusal_retries_original_lease_without_prompt_failure(self): + for concurrency in (1, 3): + with self.subTest(concurrency=concurrency), tempfile.TemporaryDirectory() as root: + backend = _FakeMooncakeStore() + store = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + attempts = [] + refusal_count = 4 + + def post(url, json_body, timeout): + if json_body.get("action") == "begin_batch": + attempts.append(tuple(item["attempt_id"] for item in json_body["tasks"])) + if len(attempts) <= refusal_count: + return dict(error="full", error_code="capture_capacity_declined", reservation_granted=False) + return server(url, json_body, timeout) + + adapter = _adapter(store, post) + adapter.hs_capture = adapter.hs_prefix_cache = True + from specforge.inference.capture_protocol import CaptureProtocol + adapter._capture_protocol = CaptureProtocol(adapter.base_url, post, 1) + channel = StreamingRefChannel(os.path.join(root, "refs")) + workers, drive = _build([adapter], _prompts(3), store, channel, lease=1, producer_concurrency=concurrency, max_worker_failures=1, max_prompt_attempts=1, feature_store_max_resident_bytes=1 << 20, backpressure_poll_s=.001, peer_wait_timeout_s=2) + self.assertEqual(drive(), 3) + self.assertTrue(all(item == attempts[0] for item in attempts[:refusal_count + 1])) + self.assertEqual(workers[0].controller.status()["prompts_failed"], 0) + self.assertEqual(workers[0].health()["in_flight"], 0) + self.assertFalse(store._capture_pending_begin) + self.assertEqual(channel.published, 3) + + +class FencedReceiveDrain(unittest.TestCase): + def make_ref(self): + backend = _FakeMooncakeStore() + producer = MooncakeFeatureStore(store=backend, store_id="run0") + server = TransferServer(backend) + adapter = SGLangServerCaptureAdapter("http://server", producer, run_id="run0", algorithm="eagle3", schema=_capture_schema("eagle3"), post_fn=server, control_fn=server, hs_capture=True, hs_prefix_cache=True) + ref = adapter.produce_refs([_task(0, 5)], capture=_eagle3_contract())[0] + return backend, ref + + def test_failed_fenced_receive_keeps_buffer_registration_and_read_authority(self): + for failure in (-800, RuntimeError("late native read")): + with self.subTest(failure=failure): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0") + original_keys = set(backend._d) + effect = failure if isinstance(failure, Exception) else None + with mock.patch.object(backend, "get_into", side_effect=effect, return_value=failure) as read, mock.patch.object(backend, "unregister_buffer", wraps=backend.unregister_buffer) as unregister: + with self.assertRaises((KeyError, RuntimeError)): + consumer.get(ref) + self.assertEqual(read.call_count, 1) + self.assertEqual(unregister.call_count, 0) + self.assertEqual(consumer.health()["capture_failed_reads"], 1) + self.assertEqual(len(consumer._quarantined_buffers), 1) + self.assertEqual(len(consumer._capture_quarantined_registrations), 1) + with self.assertRaisesRegex(RuntimeError, "read ownership|readers|not drained"): + consumer.drain_pending_removals(retry_interval_s=0) + consumer.close() # Injected transport is still owned elsewhere. + self.assertEqual(consumer.health()["capture_failed_reads"], 1) + self.assertEqual(set(backend._d), original_keys) + + def test_owned_transport_close_is_required_before_releasing_failed_read(self): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0") + with mock.patch.object(backend, "get_into", return_value=-800), self.assertRaises(KeyError): + consumer.get(ref) + consumer._owns_store = True + with mock.patch.object(backend, "close", create=True, side_effect=RuntimeError("not stopped")): + with self.assertRaisesRegex(RuntimeError, "not stopped"): + consumer.close() + self.assertEqual(consumer.health()["capture_failed_reads"], 1) + with mock.patch.object(backend, "close", create=True, return_value=0): + consumer.close() + self.assertFalse(consumer._capture_read_intents) + self.assertFalse(consumer._quarantined_buffers) + self.assertFalse(consumer._capture_quarantined_registrations) + + + def test_close_fences_new_pageable_and_pooled_reads_before_transport_shutdown(self): + for kind in ("pageable", "pinned"): + with self.subTest(kind=kind): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0", receive_buffers=kind, receive_pool_bytes=4096) + consumer._owns_store = True + entered, finish = threading.Event(), threading.Event() + def close_transport(): + entered.set() + if not finish.wait(3): + raise TimeoutError("close test gate") + return 0 + with mock.patch.object(backend, "close", create=True, side_effect=close_transport), mock.patch.object(backend, "get_into", wraps=backend.get_into) as get, ThreadPoolExecutor(max_workers=1) as pool: + closed = pool.submit(consumer.close) + try: + self.assertTrue(entered.wait(2)) + with self.assertRaisesRegex(RuntimeError, "closing"): + consumer.get(ref) + self.assertEqual(get.call_count, 0) + self.assertFalse(consumer._capture_read_intents) + finally: + finish.set() + closed.result(timeout=3) + self.assertTrue(consumer._closed) + + def test_close_refuses_an_existing_receive_and_keeps_new_reads_fenced(self): + backend, ref = self.make_ref() + consumer = MooncakeFeatureStore(store=backend, store_id="run0") + consumer._owns_store = True + entered, finish = threading.Event(), threading.Event() + original = backend.get_into + def get_into(*args): + entered.set() + if not finish.wait(3): + raise TimeoutError("read test gate") + return original(*args) + with mock.patch.object(backend, "get_into", side_effect=get_into), mock.patch.object(backend, "close", create=True, return_value=0) as close, ThreadPoolExecutor(max_workers=1) as pool: + read = pool.submit(consumer.get, ref) + try: + self.assertTrue(entered.wait(2)) + with self.assertRaisesRegex(RuntimeError, "live readers"): + consumer.close() + close.assert_not_called() + with self.assertRaisesRegex(RuntimeError, "closing"): + consumer.get(ref) + finally: + finish.set() + _, handle = read.result(timeout=3) + consumer.release(handle) + consumer.close() + self.assertEqual(close.call_count, 1) + + +class NativeHSLaunch(unittest.TestCase): + def test_managed_launch_exports_explicit_protocol_budget_and_keeps_guards(self): + from tests.test_runtime.test_launch_plan import _managed_raw, _build_managed + raw = _managed_raw(protocol="tcp") + config = raw["deployment"]["disaggregated"] + config.update(hs_capture=True, hs_prefix_cache=True, hs_chunk_tokens=384, hs_output_bytes=987654) + capture = _build_managed(raw).services[1].command + self.assertEqual(capture.env["SPECFORGE_HS_CAPTURE"], "1") + self.assertEqual(capture.env["SPECFORGE_HS_CACHE"], "1") + self.assertEqual(capture.env["SPECFORGE_HS_CHUNK_TOKENS"], "384") + self.assertEqual(capture.env["SPECFORGE_HS_OUTPUT_BYTES"], "987654") + self.assertEqual(capture.env["SGLANG_SPEC_CAPTURE_GPU_PUT"], "0") + self.assertIsNone(capture.env["FLASHINFER_DISABLE_VERSION_CHECK"]) + config.update(hs_capture=False, hs_prefix_cache=False) + native = _build_managed(raw).services[1].command + self.assertEqual(native.env["SPECFORGE_HS_CAPTURE"], "0") + self.assertEqual(native.env["SPECFORGE_HS_CACHE"], "0") diff --git a/tests/test_runtime/test_online_terminal_cleanup.py b/tests/test_runtime/test_online_terminal_cleanup.py new file mode 100644 index 0000000000..3573127cd6 --- /dev/null +++ b/tests/test_runtime/test_online_terminal_cleanup.py @@ -0,0 +1,226 @@ +"""Producer terminal scan reclaims handed-off refs only after proven success.""" +import ctypes +import os +import tempfile +import unittest +from pathlib import Path +from dataclasses import replace +from unittest import mock + +import torch + +from specforge.algorithms.builtin import builtin_algorithm_registry +from specforge.config import Config +from specforge.inference.capture_protocol import attempt_keys +from specforge.runtime.contracts import FeatureSpec, SampleRef +from specforge.runtime.data_plane.mooncake_store import MooncakeFeatureStore +from specforge.runtime.data_plane.streaming_ref_channel import StreamingRefChannel +from specforge.training.disaggregated import _build_online, _cleanup_online_published_refs +from tests.test_runtime.test_server_capture import _FakeMooncakeStore + + +class Backend(_FakeMooncakeStore): + def __init__(self): + super().__init__() + self.close_calls = 0 + self.remove_keys = [] + self.fail_remove = False + + def remove(self, key, **kwargs): + self.remove_keys.append(key) + return -1 if self.fail_remove else super().remove(key) + + def close(self): + self.close_calls += 1 + + +class TerminalCleanupTests(unittest.TestCase): + def setUp(self): + self.backend = Backend() + self.store = MooncakeFeatureStore(store_id='store', store=self.backend) + self.names = ('hidden', 'target', 'input_ids', 'loss_mask') + + def ref(self, sid, token): + wire = dict(protocol_version=1, store_id='store', sample_id=sid, gen=1, + server_epoch='epoch', attempt_id=token + '-attempt', reservation_token=token, + object_key_layout='attempt-v1', publication_ownership='attempt-v1', + writer_drained=True, state='PUBLISHED', control_url='http://forbidden/control') + wire['physical_keys'] = attempt_keys(wire, self.names) + data = torch.ones(1, 2, dtype=torch.float32) + raw = ctypes.string_at(data.data_ptr(), data.numel() * data.element_size()) + self.backend._d.update({key: raw for key in wire['physical_keys'].values()}) + return SampleRef(sample_id=sid, run_id='run', source_task_id=sid, + feature_store_uri='mooncake://store/' + sid, feature_keys=wire['physical_keys'], + feature_specs={name: FeatureSpec(name=name, shape=(1, 2), dtype='float32') for name in self.names}, + strategy='dflash', schema_version=1, target_model_version='target', tokenizer_version='tok', + num_tokens=2, estimated_bytes=32, metadata={'generation': 1, 'capture_fence': wire}) + + def handoff(self, ref): + self.store.adopt(ref) + self.store.relinquish_published(ref) + + def test_terminal_api_rejects_missing_success_or_closed_proof(self): + ref = self.ref('late', 'late-token') + self.handoff(ref) + for done, closed in ((False, False), (True, False), (False, True), (1, True)): + with self.subTest(done=done, closed=closed), self.assertRaisesRegex(RuntimeError, 'unresolved'): + self.store.cleanup_terminal_refs([ref], consumer_done=done, producer_closed=closed) + self.assertEqual(len(self.backend._d), 4) + self.assertIsNone(self.store._terminal_cleanup_authority) + self.assertEqual(self.backend.close_calls, 0) + + def test_terminal_scan_preserves_same_id_descendant_not_in_manifest(self): + old = self.ref('same', 'old') + self.handoff(old) + new = self.ref('same', 'new') + self.store.adopt(new) + new_keys = set(new.feature_keys.values()) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(str(Path(directory) / 'refs')) + channel.publish(old) + channel.mark_consumer_done() + channel.close() + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('terminal control RPC')): + _cleanup_online_published_refs(self.store, channel) + _cleanup_online_published_refs(self.store, channel) # Repeated scan is idempotent. + self.assertEqual(set(self.backend._d), new_keys) + self.assertEqual(self.store._capture_fences['same'].token, 'new') + self.assertEqual(self.backend.close_calls, 0) + + def test_failure_sentinel_wins_over_done_and_preserves_handoff_refs(self): + ref = self.ref('late', 'late') + self.handoff(ref) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(str(Path(directory) / 'refs')) + channel.publish(ref) + channel.mark_consumer_done() + channel.mark_consumer_failed('rank may still own a reader') + channel.close() + self.assertFalse(channel.consumer_done()) + self.assertTrue(channel.consumer_stopped()) + with self.assertRaisesRegex(RuntimeError, 'unresolved'): + _cleanup_online_published_refs(self.store, channel) + self.assertEqual(len(self.backend._d), 4) + self.assertEqual(self.backend.remove_keys, []) + + def test_terminal_identity_mismatch_never_falls_back_to_legacy_keys(self): + ref = self.ref('sample', 'token') + self.handoff(ref) + for bad in (replace(ref, sample_id='other'), + replace(ref, metadata={**ref.metadata, 'generation': 2})): + with self.subTest(sid=bad.sample_id, generation=bad.metadata['generation']), self.assertRaisesRegex(RuntimeError, 'identity mismatch'): + self.store.cleanup_terminal_refs([bad], consumer_done=True, producer_closed=True) + self.assertEqual(self.backend.remove_keys, []) + self.assertEqual(len(self.backend._d), 4) + + def test_terminal_uri_and_repeated_bad_proofs_are_validated_before_skip(self): + ref = self.ref('sample', 'token') + self.handoff(ref) + bad_uri = replace(ref, feature_store_uri='mooncake://different-store/sample') + with self.assertRaisesRegex(RuntimeError, 'URI identity mismatch'): + self.store.cleanup_terminal_refs([bad_uri], consumer_done=True, producer_closed=True) + self.assertEqual(self.backend.remove_keys, []) + self.assertIsNone(self.store._terminal_cleanup_authority) + self.store.cleanup_terminal_refs([ref], consumer_done=True, producer_closed=True) + removed = list(self.backend.remove_keys) + bad_generation = replace(ref, metadata={**ref.metadata, 'generation': 2}) + bad_state = replace(ref, metadata={**ref.metadata, 'capture_fence': { + **ref.metadata['capture_fence'], 'state': 'DRAINING'}}) + for bad in (bad_uri, bad_generation, bad_state): + with self.assertRaisesRegex(RuntimeError, 'unresolved'): + self.store.cleanup_terminal_refs([bad], consumer_done=True, producer_closed=True) + self.assertEqual(self.backend.remove_keys, removed) + + def test_missing_producer_eof_refuses_actual_terminal_scan(self): + ref = self.ref('late', 'late') + self.handoff(ref) + with tempfile.TemporaryDirectory() as directory: + channel = StreamingRefChannel(str(Path(directory) / 'refs')) + channel.publish(ref) + channel.mark_consumer_done() + with self.assertRaisesRegex(RuntimeError, 'closed producer'): + _cleanup_online_published_refs(self.store, channel) + self.assertEqual(self.backend.remove_keys, []) + self.assertEqual(len(self.backend._d), 4) + + def test_failed_delete_keeps_cleanup_authority_and_retries_exact_keys(self): + ref = self.ref('late', 'late') + self.handoff(ref) + self.backend.fail_remove = True + drain_samples = MooncakeFeatureStore.drain_sample_removals + drain_pending = MooncakeFeatureStore.drain_pending_removals + def bounded_samples(store, ids): + return drain_samples(store, ids, max_attempts=1, retry_interval_s=0) + def bounded_pending(store): + return drain_pending(store, max_attempts=1, retry_interval_s=0) + with mock.patch.object(MooncakeFeatureStore, 'drain_sample_removals', bounded_samples), mock.patch.object(MooncakeFeatureStore, 'drain_pending_removals', bounded_pending): + with self.assertRaisesRegex(RuntimeError, 'unresolved'): + self.store.cleanup_terminal_refs([ref], consumer_done=True, producer_closed=True) + authority = self.store._terminal_cleanup_authority + self.assertIn(ref.sample_id, authority._release_pending) + self.assertEqual(len(self.backend._d), 4) + self.backend.fail_remove = False + with self.assertRaises(RuntimeError): + self.store.cleanup_terminal_refs([ref], consumer_done=False, producer_closed=True) + self.assertEqual(len(self.backend._d), 4) + self.store.cleanup_terminal_refs([ref], consumer_done=True, producer_closed=True) + self.assertIs(self.store._terminal_cleanup_authority, authority) + self.assertFalse(authority._release_pending) + self.assertFalse(self.backend._d) + self.assertEqual(self.backend.close_calls, 0) + + def test_real_produce_terminal_scan_removes_33_late_refs_and_accepts_already_acked(self): + refs = [self.ref(str(index), 'token-' + str(index)) for index in range(34)] + for ref in refs: + self.handoff(ref) + consumer = MooncakeFeatureStore(store_id='store', store=self.backend) + with mock.patch('specforge.inference.capture_protocol.http_control', side_effect=AssertionError('normal/terminal control RPC')): + _, handle = consumer.get(refs[0]) + consumer.release(handle) + self.assertEqual(len(self.backend._d), 33 * 4) + with tempfile.TemporaryDirectory() as directory: + channel_path = str(Path(directory) / 'refs') + cfg = Config.model_validate({ + 'model': {'target_model_path': 'target', 'draft_model_config': 'draft'}, + 'data': {'prompts_path': 'unused'}, + 'training': {'strategy': 'dflash', 'role': 'producer', 'max_steps': 30, 'total_steps': 1197}, + 'deployment': {'mode': 'disaggregated', 'disaggregated': { + 'control_dir': directory, 'backend': 'mooncake', 'server_urls': ['http://capture']}}}) + def drive(*, should_stop): + self.assertFalse(should_stop()) + channel = StreamingRefChannel(channel_path) + channel.publish(refs[0]) + channel.mark_consumer_done() + # Match the observed race: completed captures are published + # after consumer_done, then workers finish and EOF is sealed. + for ref in refs[1:]: + channel.publish(ref) + channel.close() + return len(refs) + with ( + mock.patch.dict(os.environ, {'DISAGG_REF_CHANNEL': channel_path}), + mock.patch('specforge.training.disaggregated._mooncake_store', return_value=self.store), + mock.patch('specforge.training.assembly._load_input_tools'), + mock.patch('specforge.training.model_loading.resolve_draft_config'), + mock.patch('specforge.training.disaggregated._producer_capture_metadata', return_value=([1], 2, 4, 4)), + mock.patch('specforge.inference.adapters.server_capture.SGLangServerCaptureAdapter'), + mock.patch('specforge.launch.build_disagg_online_producer', return_value=([], drive)), + ): + run = _build_online(cfg, algorithm=builtin_algorithm_registry().resolve('dflash'), + build_model_bundle=mock.Mock(), + prepare_prompts=lambda *args, **kwargs: [{'task_id': 'p', 'payload': {'input_ids': [1, 2]}}], + optimizer_factory=mock.Mock(), logger=None) + self.assertEqual(run.run(), 34) + self.assertFalse(self.backend._d) + self.assertEqual(self.backend.close_calls, 0) + authority = self.store._terminal_cleanup_authority + self.assertIsNotNone(authority) + self.assertFalse(authority._owns_store) + self.assertFalse(authority._release_pending) + self.assertFalse(authority._capture_fences) + with self.assertRaises(KeyError): + self.store.adopt(refs[-1]) # Normal handoff tombstones were not reopened. + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_runtime/test_server_capture_gate.py b/tests/test_runtime/test_server_capture_gate.py index 495e1f093d..8065682dc7 100644 --- a/tests/test_runtime/test_server_capture_gate.py +++ b/tests/test_runtime/test_server_capture_gate.py @@ -178,11 +178,11 @@ def setUpClass(cls): env = dict( os.environ, - FLASHINFER_DISABLE_VERSION_CHECK="1", MOONCAKE_LOCAL_HOSTNAME=os.environ.get( "MOONCAKE_LOCAL_HOSTNAME", "127.0.0.1" ), ) + env.pop("FLASHINFER_DISABLE_VERSION_CHECK", None) if SERVER_CAPTURE == "plugin": env["SPECFORGE_SPEC_CAPTURE"] = "1" server_python = os.environ.get("SPECFORGE_CAPTURE_GATE_SERVER_PYTHON")