Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ dist/
.env
scripts/results/
*.so
# Local benchmark stores and downloaded run bundles.
.spindle-benchmarks/
benchmark-results/
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,13 @@ Notable changes to Spindle, following [Keep a Changelog](https://keepachangelog.

## Unreleased

### Added

- Synthetic monitor benchmark bundles retain raw recipe events and stock SDK result metrics, including partial failed runs, with optional uploads to a separate collector. Run-attributed OTLP points and spans can share a long-lived server resource.

- Forward-only eval and control-plane latency benchmarks, a dense/MoE CI matrix, a nightly async RL replay workflow template, and capture-to-sampler-ready timing. Benchmark jobs can target a separate results collector.
- Benchmark suites, a local results browser with JSON uploads and revision comparisons, and an OTLP receiver. Opt-in benchmark telemetry captures native sampler metrics, trainer results, backend phases, and GPU memory statistics.

## 0.1.7 - 2026-11-15

See what each run or team costs on a shared deployment: Spindle records the tokens every session uses, serves them through Tinker's `get_billing_usage`, and `spindle usage` turns them into dollars per run or label. See [Cost attribution](docs/cost-attribution.md).
Expand Down
8 changes: 8 additions & 0 deletions benchmarks/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Spindle benchmarks

Workloads and configs live here. The runner and local dashboard ship with Spindle.
See [the benchmark guide](../docs/benchmarking.md) for setup, telemetry, result
uploads, and comparison rules. No shared dashboard or credentials are required.

Run `python -m spindle.benchmarks validate benchmarks/configs/trainer.json` before
starting a GPU run. Copy a config to add a workload or tune a deployment.
39 changes: 39 additions & 0 deletions benchmarks/configs/async-rl-dapo-baseline.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"name": "qwen35b-dapo-baseline",
"suite": "async-rl",
"recipe": "dapo",
"model": "Qwen/Qwen3.6-35B-A3B",
"deployment": "src/spindle/configs/qwen36_35b_a3b_lora_32k.py",
"clients": [
2
],
"context_lengths": [
8192
],
"output_tokens": 1024,
"steps": 5,
"rank": 32,
"recipe_config": {
"groups_per_update": 1,
"samples_per_group": 4,
"max_candidate_groups": 4,
"warmup_samples": 1,
"tokenizer_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"router_replay": false,
"score_centering": false,
"top_k": 64,
"top_p": 0.95
},
"deployment_overrides": {
"trainer_max_clients_per_instance": 2,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"trainer_gpu": "H200",
"inference_gpu": "H200",
"miles_cfg.cli_options.use_rollout_routing_replay": false,
"sglang_cfg.enable_return_routed_experts": false
},
"experiment": "nightly-dapo-replay",
"timeout_s": 14400
}
39 changes: 39 additions & 0 deletions benchmarks/configs/async-rl-dapo-replay.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"name": "qwen35b-dapo-replay",
"suite": "async-rl",
"recipe": "dapo",
"model": "Qwen/Qwen3.6-35B-A3B",
"deployment": "src/spindle/configs/qwen36_35b_a3b_lora_32k.py",
"clients": [
2
],
"context_lengths": [
8192
],
"output_tokens": 1024,
"steps": 5,
"rank": 32,
"recipe_config": {
"groups_per_update": 1,
"samples_per_group": 4,
"max_candidate_groups": 4,
"warmup_samples": 1,
"tokenizer_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"router_replay": true,
"score_centering": true,
"top_k": 64,
"top_p": 0.95
},
"deployment_overrides": {
"trainer_max_clients_per_instance": 2,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"trainer_gpu": "H200",
"inference_gpu": "H200",
"miles_cfg.cli_options.use_rollout_routing_replay": true,
"sglang_cfg.enable_return_routed_experts": true
},
"experiment": "nightly-dapo-replay",
"timeout_s": 14400
}
39 changes: 39 additions & 0 deletions benchmarks/configs/async-rl-dapo-router.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"name": "qwen35b-dapo-router",
"suite": "async-rl",
"recipe": "dapo",
"model": "Qwen/Qwen3.6-35B-A3B",
"deployment": "src/spindle/configs/qwen36_35b_a3b_lora_32k.py",
"clients": [
2
],
"context_lengths": [
8192
],
"output_tokens": 1024,
"steps": 5,
"rank": 32,
"recipe_config": {
"groups_per_update": 1,
"samples_per_group": 4,
"max_candidate_groups": 4,
"warmup_samples": 1,
"tokenizer_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"router_replay": true,
"score_centering": false,
"top_k": 64,
"top_p": 0.95
},
"deployment_overrides": {
"trainer_max_clients_per_instance": 2,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"trainer_gpu": "H200",
"inference_gpu": "H200",
"miles_cfg.cli_options.use_rollout_routing_replay": true,
"sglang_cfg.enable_return_routed_experts": true
},
"experiment": "nightly-dapo-replay",
"timeout_s": 14400
}
39 changes: 39 additions & 0 deletions benchmarks/configs/async-rl-dapo-score-centering.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"name": "qwen35b-dapo-score-centering",
"suite": "async-rl",
"recipe": "dapo",
"model": "Qwen/Qwen3.6-35B-A3B",
"deployment": "src/spindle/configs/qwen36_35b_a3b_lora_32k.py",
"clients": [
2
],
"context_lengths": [
8192
],
"output_tokens": 1024,
"steps": 5,
"rank": 32,
"recipe_config": {
"groups_per_update": 1,
"samples_per_group": 4,
"max_candidate_groups": 4,
"warmup_samples": 1,
"tokenizer_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"router_replay": false,
"score_centering": true,
"top_k": 64,
"top_p": 0.95
},
"deployment_overrides": {
"trainer_max_clients_per_instance": 2,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7",
"trainer_gpu": "H200",
"inference_gpu": "H200",
"miles_cfg.cli_options.use_rollout_routing_replay": false,
"sglang_cfg.enable_return_routed_experts": false
},
"experiment": "nightly-dapo-replay",
"timeout_s": 14400
}
14 changes: 14 additions & 0 deletions benchmarks/configs/async-rl-dapo.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"name": "qwen27b-dapo",
"suite": "async-rl",
"recipe": "dapo",
"model": "Qwen/Qwen3.8-27B",
"deployment": "src/spindle/configs/qwen38_27b_lora_64k.py",
"clients": [2],
"context_lengths": [8192],
"output_tokens": 1024,
"steps": 5,
"rank": 32,
"recipe_config": {"groups_per_update": 1, "samples_per_group": 4, "max_candidate_groups": 4, "warmup_samples": 1},
"deployment_overrides": {"trainer_max_clients_per_instance": 2, "inference_min_replicas": 0, "inference_max_replicas": 1}
}
14 changes: 14 additions & 0 deletions benchmarks/configs/async-rl-swe-gym.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"name": "qwen9b-swe-gym",
"suite": "async-rl",
"recipe": "swe-gym",
"model": "Qwen/Qwen3.5-9B",
"deployment": "src/spindle/configs/qwen35_9b_lora_128k.py",
"clients": [2],
"context_lengths": [32768],
"output_tokens": 8192,
"steps": 5,
"rank": 32,
"timeout_s": 14400,
"deployment_overrides": {"trainer_max_clients_per_instance": 2, "inference_min_replicas": 0, "inference_max_replicas": 1}
}
28 changes: 28 additions & 0 deletions benchmarks/configs/ci-large-moe-eval.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "ci-large-moe-eval",
"experiment": "ci-large-moe",
"suite": "eval",
"model": "zai-org/GLM-5.3",
"deployment": "src/spindle/configs/glm53_lora_256k.py",
"clients": [
1,
2,
4
],
"context_lengths": [
1024,
8192
],
"steps": 10,
"warmup_steps": 2,
"batch_size": 1,
"timeout_s": 14400,
"deployment_overrides": {
"trainer_gpu": "B300",
"inference_gpu": "B300",
"trainer_max_clients_per_instance": 4,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "ba32f8c19c8738e2c57c71798c601d477a7bf844"
}
}
28 changes: 28 additions & 0 deletions benchmarks/configs/ci-large-moe-trainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "ci-large-moe-trainer",
"experiment": "ci-large-moe",
"suite": "trainer",
"model": "zai-org/GLM-5.3",
"deployment": "src/spindle/configs/glm53_lora_256k.py",
"clients": [
1,
2,
4
],
"context_lengths": [
1024,
8192
],
"steps": 10,
"warmup_steps": 2,
"batch_size": 1,
"timeout_s": 14400,
"deployment_overrides": {
"trainer_gpu": "B300",
"inference_gpu": "B300",
"trainer_max_clients_per_instance": 4,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "ba32f8c19c8738e2c57c71798c601d477a7bf844"
}
}
28 changes: 28 additions & 0 deletions benchmarks/configs/ci-small-dense-eval.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "ci-small-dense-eval",
"experiment": "ci-small-dense",
"suite": "eval",
"model": "Qwen/Qwen3.5-9B",
"deployment": "src/spindle/configs/qwen35_9b_instruct_lora_16k.py",
"clients": [
1,
2,
4
],
"context_lengths": [
1024,
8192
],
"steps": 10,
"warmup_steps": 2,
"batch_size": 1,
"timeout_s": 14400,
"deployment_overrides": {
"trainer_gpu": "H100",
"inference_gpu": "H100",
"trainer_max_clients_per_instance": 4,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a"
}
}
28 changes: 28 additions & 0 deletions benchmarks/configs/ci-small-dense-trainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "ci-small-dense-trainer",
"experiment": "ci-small-dense",
"suite": "trainer",
"model": "Qwen/Qwen3.5-9B",
"deployment": "src/spindle/configs/qwen35_9b_instruct_lora_16k.py",
"clients": [
1,
2,
4
],
"context_lengths": [
1024,
8192
],
"steps": 10,
"warmup_steps": 2,
"batch_size": 1,
"timeout_s": 14400,
"deployment_overrides": {
"trainer_gpu": "H100",
"inference_gpu": "H100",
"trainer_max_clients_per_instance": 4,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a"
}
}
28 changes: 28 additions & 0 deletions benchmarks/configs/ci-small-moe-eval.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "ci-small-moe-eval",
"experiment": "ci-small-moe",
"suite": "eval",
"model": "Qwen/Qwen3.6-35B-A3B",
"deployment": "src/spindle/configs/qwen36_35b_a3b_lora_32k.py",
"clients": [
1,
2,
4
],
"context_lengths": [
1024,
8192
],
"steps": 10,
"warmup_steps": 2,
"batch_size": 1,
"timeout_s": 14400,
"deployment_overrides": {
"trainer_gpu": "H200",
"inference_gpu": "H200",
"trainer_max_clients_per_instance": 4,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7"
}
}
28 changes: 28 additions & 0 deletions benchmarks/configs/ci-small-moe-trainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "ci-small-moe-trainer",
"experiment": "ci-small-moe",
"suite": "trainer",
"model": "Qwen/Qwen3.6-35B-A3B",
"deployment": "src/spindle/configs/qwen36_35b_a3b_lora_32k.py",
"clients": [
1,
2,
4
],
"context_lengths": [
1024,
8192
],
"steps": 10,
"warmup_steps": 2,
"batch_size": 1,
"timeout_s": 14400,
"deployment_overrides": {
"trainer_gpu": "H200",
"inference_gpu": "H200",
"trainer_max_clients_per_instance": 4,
"inference_min_replicas": 0,
"inference_max_replicas": 1,
"weights_revision": "345d467b117d1fd5eda9b879a5988de9ae5f47a7"
}
}
Loading
Loading