Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions .claude/skills/capture-trace/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,16 @@ ATOM annotates the critical path with `torch.profiler.record_function`. The **ki
| `dummy_decode[...]` | **DP-sync dummy** (idle rank keeps the MoE collective aligned) |
| `dummy_eager_decode[...]` | DP-sync dummy, forced eager |
| `dummy_prefill[...]` | warmup dummy prefill |
| `draft[i/k bs=]` | eagle/MTP draft mid-step |
| `propose_eagle[i/k tok= bs=<real>/<pad> (graph)]` | eagle/MTP draft step `i` of `k` |
| `propose_dspark[bs=<real>/<pad> T= (graph)]` | DSpark block draft (one parallel pass) |
| `draft_kv[bs= tok=]` | DSpark rolling-window KV write, after every target forward |
| `dspark_sched[bs=]` | DSpark confidence-schedule ell computation |

Fields: `bs` effective (real) batch — on the CUDAGraph path shown as `bs=<real>/<graph>` (e.g. `bs=117/128`) when the replayed graph is padded above the real batch; `tok` total tokens, `ctx` per-seq context lens (truncated if many), `p`/`d` prefill/decode seq counts, `spec` speculative steps, and **`tbo=1`** appended when the step ran Two-Batch-Overlap ubatches.
Every drafter's propose pass shares the `propose_` prefix, so one grep covers all flavors; `draft_kv` and `dspark_sched` keep their own names because neither is a propose. The `/<pad>` field marks a pass that *can* pad — eagle's step 0 cannot, so it carries no field at all — and padding actually happened only when the two numbers differ (`bs=44/44` is a pass that declined). The trailing ` graph` is the one that appears only on a replay, which makes "did the draft get into a graph" a grep rather than an inference.

`bs` and `tok` are different counts on purpose: `bs` is sequences, `tok` the rows of that particular forward. An eagle step 0 runs the whole token stream while steps 1+ run one row per sequence, and the draft's shape-driven JIT tracks `tok`.

Fields: `bs` effective (real) batch — on the CUDAGraph path shown as `bs=<real>/<graph>` (e.g. `bs=117/128`), the second number being the shape actually run, equal to the first when nothing was padded; `tok` total tokens, `ctx` per-seq context lens (truncated if many), `p`/`d` prefill/decode seq counts, `spec` speculative steps, and **`tbo=1`** appended when the step ran Two-Batch-Overlap ubatches.

Distinguishing dummy vs real matters: e.g. the leading `dummy_decode[bs=1 tok=1]` runs are DP-sync idle steps, NOT real decode. `parse_trace.py` matches only the exact `prefill[` / `decode[` prefixes, so dummy/eager variants are auto-excluded from its stats.

Expand Down
51 changes: 27 additions & 24 deletions atom/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1545,7 +1545,6 @@ class Config:
# sizes the process group and the token collectives.
dp_logical_size: int = 0
master_addr: str = "127.0.0.1"
graph_bs: list[int] | None = None
enable_dp_attention: bool = False
# DP request-routing strategy used by CoreManager to pick an engine rank:
# "round_robin" | "least_requests" (default) | "least_tokens". Only has an
Expand Down Expand Up @@ -1634,17 +1633,22 @@ def tp_world_size(self) -> int:
visible = torch.cuda.device_count()
return visible if 0 < visible < tp else tp

def _set_cudagraph_sizes(self):
if self.compilation_config.cudagraph_capture_sizes:
self.graph_bs = self.compilation_config.cudagraph_capture_sizes
else:
cuda_graph_sizes = self.compilation_config.cuda_graph_sizes
if len(cuda_graph_sizes) == 1:
self.graph_bs = [1, 2, 4, 8] + [
i for i in range(16, cuda_graph_sizes[0] + 1, 16)
]
elif len(cuda_graph_sizes) > 1:
self.graph_bs = cuda_graph_sizes
@property
def capture_sizes(self) -> list[int]:
"""The declared CUDAGraph capture ladder, in batch sizes.

Declared, not schedulable -- `ModelRunner` drops what its own token
budget can never produce, a bound needing the drafter's resolved
`mtp_k` that config cannot see. Returns a copy: the runner sorts and
filters in place.
"""
declared = self.compilation_config.cudagraph_capture_sizes
if declared:
return list(declared)
sizes = self.compilation_config.cuda_graph_sizes
if len(sizes) == 1:
return [1, 2, 4, 8] + list(range(16, sizes[0] + 1, 16))
return list(sizes)

def __post_init__(self):
self.moe_backend = self.moe_backend.strip().lower()
Expand Down Expand Up @@ -1865,18 +1869,17 @@ def __post_init__(self):
# only for server mode or plugin mode(vllm)
# for torch compile policy, plugin mode(vllm) uses the ATOM compile policy
# for cuda graph capture, plugin mode(vllm) uses the vLLM's cuda graph capture policy
if not is_plugin_mode() or (
self.plugin_config is not None and self.plugin_config.is_vllm
):
if self.compilation_config.level == CompilationLevel.PIECEWISE:
self.compilation_config.set_splitting_ops_for_v1()
self._set_cudagraph_sizes()
# Keep an explicit cudagraph_mode (e.g. FULL); default to
# PIECEWISE only when unset. splitting_ops/sizes are set either
# way so the model is still piece-split-compiled at level 3.
if self.compilation_config.cudagraph_mode is None:
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
self.compilation_config.init_with_cudagraph_sizes()
if (
not is_plugin_mode()
or (self.plugin_config is not None and self.plugin_config.is_vllm)
) and self.compilation_config.level == CompilationLevel.PIECEWISE:
self.compilation_config.set_splitting_ops_for_v1()
# Keep an explicit cudagraph_mode (e.g. FULL); default to
# PIECEWISE only when unset. splitting_ops/sizes are set either
# way so the model is still piece-split-compiled at level 3.
if self.compilation_config.cudagraph_mode is None:
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
self.compilation_config.init_with_cudagraph_sizes()

self.torch_dtype = (
self.hf_config.dtype
Expand Down
Loading
Loading