Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@ Notable changes to Spindle, following [Keep a Changelog](https://keepachangelog.

## Unreleased

### Fixed

- Long-sequence training requests with expert-routing replay no longer fail with intermittent 503 errors.

## 0.1.7 - 2026-11-15

See what each run or team costs on a shared deployment: Spindle records the tokens every session uses, serves them through Tinker's `get_billing_usage`, and `spindle usage` turns them into dollars per run or label. See [Cost attribution](docs/cost-attribution.md).
Expand Down
14 changes: 7 additions & 7 deletions src/spindle/control_plane/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -775,16 +775,16 @@ async def forward_backward(request: Request) -> dict[str, str]:
encoding=request.headers.get("content-encoding", "identity"),
)
content_type = request.headers.get("content-type", "application/json")
body, identity = await asyncio.to_thread(
_training_envelope,
body,
content_type,
request.headers.get("content-encoding", "identity"),
encoding = request.headers.get("content-encoding", "identity")
decoded, identity = await asyncio.to_thread(
_training_envelope, body, content_type, encoding
)
mark("cp.forward_backward.decompressed", bytes=len(body))
mark("cp.forward_backward.decompressed", bytes=len(decoded))
del decoded
model_id = identity["model_id"]
engine = await control_plane.engine_for(model_id)
request_id = await engine.forward_backward(body, content_type)
# Decompressed replay batches reach hundreds of MB.
request_id = await engine.forward_backward(body, content_type, encoding)
mark("cp.forward_backward.forwarded", request_id=request_id, model_id=model_id)
return {"request_id": request_id, "model_id": model_id}

Expand Down
4 changes: 3 additions & 1 deletion src/spindle/engine/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,9 @@ async def accept_model(self, model_id: str, spec: object) -> bool: ...

async def model_ids(self) -> tuple[str, ...]: ...

async def forward_backward(self, body: bytes, content_type: str) -> str: ...
async def forward_backward(
self, body: bytes, content_type: str, content_encoding: str = "identity"
) -> str: ...

async def forward(self, request: dict) -> str: ...

Expand Down
21 changes: 14 additions & 7 deletions src/spindle/engine/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from typing import Any

import httpx
import zstandard
from fastapi import Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse, Response
from pydantic import BaseModel, ConfigDict
Expand All @@ -21,6 +20,10 @@
)


# Trainers can take over 30 s to ingest long-context batches; the Tinker SDK waits 60 s.
FORWARD_BACKWARD_TIMEOUT = httpx.Timeout(60.0, connect=30.0)


class AcceptModelBody(BaseModel):
model_config = ConfigDict(protected_namespaces=())

Expand Down Expand Up @@ -110,12 +113,10 @@ async def models() -> dict[str, tuple[str, ...]]:

@app.post("/api/v1/forward_backward")
async def forward_backward(request: Request) -> dict[str, str]:
body = await request.body()
if request.headers.get("content-encoding") == "zstd":
body = zstandard.ZstdDecompressor().decompress(body)
request_id = await server.forward_backward(
body,
await request.body(),
request.headers.get("content-type", "application/json"),
request.headers.get("content-encoding", "identity"),
)
return {"request_id": request_id}

Expand Down Expand Up @@ -239,11 +240,17 @@ async def model_ids(self) -> tuple[str, ...]:
_raise_mapped(response)
return tuple(response.json()["model_ids"])

async def forward_backward(self, body: bytes, content_type: str) -> str:
async def forward_backward(
self, body: bytes, content_type: str, content_encoding: str = "identity"
) -> str:
headers = {"Content-Type": content_type}
if content_encoding != "identity":
headers["Content-Encoding"] = content_encoding
response = await self.http.post(
"/api/v1/forward_backward",
content=body,
headers={"Content-Type": content_type},
headers=headers,
timeout=FORWARD_BACKWARD_TIMEOUT,
)
_raise_mapped(response)
return response.json()["request_id"]
Expand Down
9 changes: 8 additions & 1 deletion src/spindle/engine/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from dataclasses import dataclass, field
from typing import Protocol

import zstandard
from tinker.types.forward_backward_input import ForwardBackwardInput

from spindle.encoding import fingerprint
Expand Down Expand Up @@ -207,8 +208,14 @@ async def model_ids(self) -> tuple[str, ...]:
def loaded(self) -> tuple[str, ...]:
return tuple(self._models)

async def forward_backward(self, body: bytes, content_type: str) -> str:
async def forward_backward(
self, body: bytes, content_type: str, content_encoding: str = "identity"
) -> str:
mark("engine.forward_backward.received", bytes=len(body))
if content_encoding == "zstd":
body = await asyncio.to_thread(
zstandard.ZstdDecompressor().decompress, body
)
model_id, seq_id, kind, payload = await asyncio.to_thread(
decode_forward_backward, body, content_type
)
Expand Down
9 changes: 7 additions & 2 deletions tests/engine/test_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import fastapi.encoders
import httpx
import pytest
import zstandard
from tinker import types
from tinker.proto.request_conv import forward_backward_request_to_proto
from tinker.types.forward_backward_input import ForwardBackwardInput
Expand Down Expand Up @@ -126,10 +127,14 @@ async def run() -> None:
),
)
body = forward_backward_request_to_proto(request).SerializeToString()
request_id = await client.forward_backward(
body,
request_id = await client.forward_backward(body, "application/x-protobuf")
# A retry may arrive compressed; it must resolve to the same request.
retried = await client.forward_backward(
zstandard.ZstdCompressor().compress(body),
"application/x-protobuf",
"zstd",
)
assert retried == request_id
state = await client.retrieve_future(request_id, timeout=1.0)
assert state.status == FutureStatus.COMPLETE
payload = state.result["payload"]
Expand Down
Loading