diff --git a/.github/time_budget.yaml b/.github/time_budget.yaml index c9fcf02be2a2..a1e330e9f0af 100644 --- a/.github/time_budget.yaml +++ b/.github/time_budget.yaml @@ -413,8 +413,8 @@ models: bh_quietbox_2: 80 demo: bh_galaxy: 470 - bh_sc1: 1750 - bh_sc1_high_power: 388 + bh_sc1: 1865 + bh_sc1_high_power: 403 bh_sc4: 60 wh_n150: 500 wh_n150_civ2: 300 diff --git a/conftest.py b/conftest.py index f8f547a30a2b..068efd8428e9 100644 --- a/conftest.py +++ b/conftest.py @@ -17,6 +17,7 @@ import torch from loguru import logger +from models.common import timing_events from models.demos.utils.trace_region_sizes import TRACE_MODEL_KEY_PARAM, resolve_trace_region_size from models.tt_transformers.demo.trace_region_config import get_logical_sku, get_supported_trace_region_size from tests.scripts.common import get_updated_device_params, run_process_and_get_result @@ -472,11 +473,13 @@ def device(request, device_params): device.cache_entries_counter = CacheEntriesCounter(device) + timing_events.emit("device.open", fixture="device", device_id=device_id, test=request.node.nodeid) yield device # Restore the original default device BEFORE closing the test-specific one ttnn.SetDefaultDevice(original_default_device) ttnn.close_device(device) + timing_events.emit("device.close", fixture="device", test=request.node.nodeid) # Reset fabric config to DISABLED if not None, and do nothing otherwise @@ -666,6 +669,7 @@ def mesh_device(request, silicon_arch_name, device_params): mesh_device.cache_entries_counter = CacheEntriesCounter(mesh_device) logger.debug(f"multidevice with {mesh_device.get_num_devices()} devices is created") + timing_events.emit("device.open", fixture="mesh_device", shape=list(mesh_device.shape), test=request.node.nodeid) yield mesh_device for submesh in mesh_device.get_submeshes(): @@ -673,6 +677,7 @@ def mesh_device(request, silicon_arch_name, device_params): ttnn.close_mesh_device(mesh_device) reset_fabric(fabric_config) + timing_events.emit("device.close", fixture="mesh_device", test=request.node.nodeid) del mesh_device diff --git a/models/common/timing_events.py b/models/common/timing_events.py new file mode 100644 index 000000000000..1c7184ad5252 --- /dev/null +++ b/models/common/timing_events.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Grep-able timing events for device tests: one line per event, ``TT_EVENT {json}``, with a wall-clock UTC +timestamp, so a run's phases can be reconstructed from its log even though pytest prints captured output after +each test (not chronologically). Events: ``device.open`` / ``device.close`` (device fixtures), +``phase.begin`` / ``phase.end`` (test bodies: reference, oracle, weights, compute), ``cache.hit`` / +``cache.miss`` (result and weight caches, with key and bytes). Printing only: no behavior change for callers. +""" + +import json +import time +from contextlib import contextmanager +from datetime import datetime, timezone + +PREFIX = "TT_EVENT " + + +def emit(event: str, **fields) -> None: + record = {"ts": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), "event": event, **fields} + print(PREFIX + json.dumps(record, default=str), flush=True) + + +@contextmanager +def phase(kind: str, **fields): + """``phase.begin`` / ``phase.end`` (with seconds) of a ``kind`` phase (reference / oracle / weights / compute) + around a block; ``phase.end`` is emitted on failure too.""" + emit("phase.begin", phase=kind, **fields) + start = time.perf_counter() + try: + yield + finally: + emit("phase.end", phase=kind, seconds=round(time.perf_counter() - start, 3), **fields) + + +def cache(hit: bool, cache_name: str, key: str, nbytes: int | None = None) -> None: + emit("cache.hit" if hit else "cache.miss", cache=cache_name, key=key, bytes=nbytes) diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/README.md b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/README.md new file mode 100644 index 000000000000..7258e758cf28 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/README.md @@ -0,0 +1,75 @@ +# DeepSeek-V4.1-Flash reference: CPU-runnable + +This directory holds a vendored copy of the DeepSeek-V4.1-Flash reference inference code. The upstream tilelang/CUDA kernels are replaced by torch ports, so the model runs on CPU with `world_size=1` and without initializing `torch.distributed`. + +## Provenance + +- Upstream: , revision + `dba1be0a40aa45a94ad051997016db3960a90277`, directory `inference/`. +- License: MIT (upstream `LICENSE`: "Copyright (c) 2023 DeepSeek"). The vendored files carry that notice as an SPDX header. + +| File | Origin | +|---|---| +| `model.py` | upstream `inference/model.py` (deviations below) | +| `engram.py` | upstream `inference/engram.py` (provenance comment only) | +| `vision.py` | upstream `inference/vision.py` (provenance comment only) | +| `image_processor.py` | upstream `inference/image_processor.py` (provenance comment only) | +| `config.json` | upstream `inference/config.json`, unchanged (the released model config) | +| `kernel_cpu.py` | new (Tenstorrent, Apache-2.0): torch ports of upstream `inference/kernel.py` | +| `testing.py` | new (Tenstorrent, Apache-2.0): small config, stub tokenizer, seeded weight init, prefill helper | +| `oracle.py` | new (Tenstorrent, Apache-2.0): cached expected results (block captures, shared and final state, chunk contract) for real-dims layer subsets, synthetic or checkpoint weights | +| `__init__.py` | new, empty | + +Upstream files that are not vendored: `kernel.py` (tilelang, replaced by `kernel_cpu.py`), `generate.py`, `convert.py`, `run.sh`, `requirements.txt`, and `examples/`. + +## Deviations from upstream (complete list) + +1. Each vendored `.py` file begins with a 2-line SPDX header naming the upstream copyright and MIT license, then a provenance comment block (3 lines, plus a blank line) that points to this README. `model.py` also carries `# isort: skip_file` and `# fmt: off` (9 header lines in total), because the repository's black and isort pre-commit hooks would otherwise reflow upstream code (assert wrapping, import order). The other vendored files are already stable under those hooks. +2. `model.py`: four import statements are changed. Everything else is byte-identical to upstream. Check with `diff <(tail -n +10 model.py) /inference/model.py`; for the other files, use `tail -n +7`. + - `from engram import ...` becomes `from .engram import ...`. The same change applies to `image_processor` and `vision`. Rationale: package-relative imports. + - `from kernel import (...)` becomes `from .kernel_cpu import (...)`. Rationale: CPU kernels. The imported names and signatures are the same. + +No other change was needed to run on CPU. `Transformer.__init__` already sets `world_size = 1` when `torch.distributed` is not initialized, and every `dist.*` call is guarded by `world_size > 1`. + +## Running it + +Follow upstream `generate.py`: construct the model and run it under `torch.set_default_dtype(torch.bfloat16)`, or use the `set_dtype(torch.bfloat16)` context manager from `model.py`. Two contracts depend on this: +- `fp8_gemm` and `fp4_gemm` return the default dtype. +- `act_quant` and `fp4_act_quant` take bf16 input only, like the tilelang kernels (`in_dtype=BF16`). + +`testing.py` wraps this in the following helpers: + +```python +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.testing import build_small_model, prefill +model = build_small_model(seed=0) # 6 backbone layers (all six block types) + 1 DSpark layer +logits, main_hidden = prefill(model, input_ids) # start_pos 0 forward + DSpark window-cache seeding +``` + +`SmallScheduleConfig` puts one layer of each block type on the schedule, in this order: SWA_ONLY, KV_INDEX_SOURCE (ratio 2), CONSUMER_RATIO2, CANDIDATE_SOURCE (ratio 1), CONSUMER_RATIO1, CANDIDATE_INDEX_SOURCE. The block types are the canonical ones from `DeepSeekV41FlashConfig.block_type`. Engram runs on layers 0 (SWA) and 1 (KV source). + +Weights are stored in checkpoint dtypes: +- dense weights: FP8 e4m3 with E8M0 32x32 block scales +- routed experts: FP4 `float4_e2m1fn_x2` with E8M0 per-32 scales +- Engram tables: FP8 with E8M0 per-32 row scales + +The Engram hash needs a compressed token map. `StubTokenizer` supplies one without a real tokenizer. + +The upstream code has module-level state: `world_size`, `default_dtype`, and the `shared_attn` runtime. So, as upstream says, run one model at a time per process. Layers run in order, and each prefill at start_pos 0 rewrites every shared slot before it is read. + +## Kernel port fidelity (`kernel_cpu.py`) + +| Function | Fidelity to tilelang | Notes | +|---|---|---| +| `act_quant` | bit-exact | per-block amax floored at 1e-4. With `scale_fmt`, the scale is `2^ceil(log2(amax * fp32(1/448)))` via the same exponent-bit trick; otherwise it is `amax * fp32(1/448)`. Values are clamped to ±448, then cast with RNE to e4m3. Inplace mode does QDQ back to bf16 | +| `fp4_act_quant` | bit-exact | E8M0 path: per-32 groups, amax floored at 6·2^-126, power-of-2 scale. E4M3 path: groups of 16, amax floored at 6·2^-9, scale is `e4m3(amax/6)`. E2M1 cast is RNE with ties to the even code. Packing puts the even element in the low nibble, matching upstream `convert.py` | +| `fp8_gemm`, `fp4_gemm` | exact block products; summation order within a block differs | Each K block (32) is summed in fp32, scaled by `a_s * b_s`, then accumulated in K order, as the kernel does. Only the order within a block differs from the tensor-core MMA, so rare 1 bf16-ulp output differences are possible | +| `sparse_attn` | same algorithm; GEMM order and `exp` differ | Online softmax over blocks of 64 indices. The running max starts at -1e30, so rows where every index is -1 give zeros. Index -1 gets logit -inf and a zeroed row. The fp32 sum of probabilities feeds the denominator; probabilities are cast to bf16 before PV. The sink is added unscaled. Output is bf16. Expected difference: about 1 bf16 ulp of the output | +| `hc_split_sinkhorn` | same fp32 operation order | Only `exp`/`sigmoid` may differ in the last fp32 ulp | + +Tests: `models/demos/deepseek_v3_d_p/tests/v41/torch/test_v41_kernels_cpu.py` checks the kernels against hand cases and independent formulations. `test_v41_reference_cpu.py` runs the small-model prefill smoke test. + +## Upstream behaviors worth knowing (not changed) + +- `Indexer.forward` calls `topk(..., sorted=False)`. How it breaks ties among equal scores is implementation-defined, so CPU and CUDA can choose different positions when scores tie exactly (for example, all-zero relu scores). +- `image_processor.prepare_vl_inputs` imports `encoding` lazily. That module is not in upstream `inference/`, so tests build `ImageInput` directly. +- The `__main__` self-test in `model.py` targets CUDA and is not used here. diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/__init__.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/__init__.py new file mode 100644 index 000000000000..296a6f8f3038 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/config.json b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/config.json new file mode 100644 index 000000000000..7a915cc69e21 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/config.json @@ -0,0 +1,66 @@ +{ + "vocab_size": 129280, + "dim": 5120, + "moe_inter_dim": 2304, + "n_layers": 40, + "n_mtp_layers": 3, + "dspark_block_size": 5, + "dspark_noise_token_id": 128799, + "dspark_target_layer_ids": [37, 38, 39], + "dspark_markov_rank": 256, + "dspark_n_routed_experts": 128, + "dspark_n_activated_experts": 3, + "n_heads": 64, + "n_routed_experts": 384, + "n_shared_experts": 1, + "n_activated_experts": 6, + "score_func": "sqrtsoftplus", + "route_scale": 1.5, + "swiglu_limit": 10.0, + "q_lora_rank": 1280, + "head_dim": 512, + "rope_head_dim": 64, + "norm_eps": 1e-20, + "o_groups": 8, + "o_lora_rank": 1024, + "window_size": 128, + "kv_source_layers": [2, 8, 14, 20], + "index_source_layers": [2, 8, 14, 20, 24, 28, 32, 36], + "original_seq_len": 65536, + "rope_theta": 10000, + "rope_factor": 16, + "beta_fast": 32, + "beta_slow": 1, + "index_n_heads": 32, + "index_head_dim": 128, + "index_topk": 512, + "candidate_source_layer": 20, + "candidate_topk_blocks": 2048, + "candidate_block_size": 8, + "hc_mult": 4, + "hc_sinkhorn_iters": 20, + "hc_eps": 1e-06, + "engram_layer_ids": [1, 14], + "engram_vocab_size": 16000000, + "engram_num_embeddings": [384006168, 384016682], + "engram_max_ngram_size": 4, + "engram_pad_id": 2, + "engram_compressed_vocab_size": 99092, + "dtype": "fp8", + "expert_dtype": "fp4", + "compress_rope_theta": 160000, + "compress_ratios": [0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], + "vision_n_layers": 32, + "vision_dim": 1024, + "vision_n_heads": 16, + "vision_inter_dim": 2816, + "vision_patch_size": 14, + "vision_downsample_ratio": 3, + "vision_max_n_token": 1024, + "vision_min_pixels": 295936, + "vision_max_wh_ratio": null, + "image_token_id": 129264, + "engram_n_heads": 8, + "engram_head_dim": 256, + "vision_rope_theta": 10000 +} diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/engram.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/engram.py new file mode 100644 index 000000000000..e3998151228e --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/engram.py @@ -0,0 +1,190 @@ +# SPDX-FileCopyrightText: © 2023 DeepSeek +# SPDX-License-Identifier: MIT +# Vendored from huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash, revision +# dba1be0a40aa45a94ad051997016db3960a90277, file inference/engram.py (MIT License). +# Deviations from upstream are listed in README.md next to this file. + +from dataclasses import dataclass + +import numpy as np +import torch +from sympy import isprime +from torch import nn + + +def find_next_prime(start: int, seen_primes: set[int]) -> int: + """The smallest prime above `start` that has not been handed out yet.""" + candidate = start + 1 + while not isprime(candidate) or candidate in seen_primes: + candidate += 1 + return candidate + + +def build_compressed_token_map(tokenizer) -> tuple[list[int], int]: + """Map every token id onto a smaller id space where tokens that normalize alike collapse together. + + N-grams are hashed over these compressed ids, so " The", "the" and "THE" all hash the same way. + Returns the lookup plus the size of the compressed vocab -- and that size matters beyond bounds + checking, because every hash multiplier is derived from it. + """ + from tokenizers import Regex, normalizers + + # a private-use char, so a token that is exactly one space survives Strip() instead of + # collapsing to the empty string and merging with unrelated tokens + sentinel = "\ue000" + normalizer = normalizers.Sequence( + [ + normalizers.NFKC(), + normalizers.NFD(), + normalizers.StripAccents(), + normalizers.Lowercase(), + normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), + normalizers.Replace(Regex(r"^ $"), sentinel), + normalizers.Strip(), + normalizers.Replace(sentinel, " "), + ] + ) + + # the raw Rust tokenizer, matching what training decodes with (no clean_up_tokenization_spaces) + backend = tokenizer.backend_tokenizer + key_to_new: dict[str, int] = {} + lookup = [0] * len(tokenizer) + for token_id in range(len(tokenizer)): + text = backend.decode([token_id], skip_special_tokens=False) + if "\ufffd" in text: + # a partial UTF-8 byte token: nothing to normalize, so key it by its raw form + key = backend.id_to_token(token_id) + else: + normalized = normalizer.normalize_str(text) + key = normalized if normalized else text + + new_id = key_to_new.get(key) + if new_id is None: + new_id = len(key_to_new) + key_to_new[key] = new_id + lookup[token_id] = new_id + + return lookup, len(key_to_new) + + +def compute_hash_multipliers( + layer_ids: tuple[int, ...], max_ngram_size: int, tokenizer_vocab_size: int +) -> torch.Tensor: + """One multiplier per (layer, lookback), from a per-layer RNG so layers hash differently. + + Kept odd, and bounded so that `token_id * multiplier` cannot overflow int64. + """ + max_long = np.iinfo(np.int64).max + multiplier_bound = max(1, (max_long // tokenizer_vocab_size) // 2) + rows = [] + for layer_id in layer_ids: + generator = np.random.default_rng(10007 * layer_id) + values = generator.integers( + low=0, + high=multiplier_bound, + size=(max_ngram_size,), + dtype=np.int64, + ) + rows.append(torch.tensor(values * 2 + 1)) + return torch.stack(rows) + + +@dataclass(frozen=True) +class EngramLayout: + """Bucket layout of the n-gram hash tables. + + A position is hashed as `max_ngram_size - 1` n-grams (2-gram .. max_ngram_size-gram), each split + over `n_heads` heads. Every (n-gram size, head) pair owns its own prime-sized bucket range in the + layer's table; the primes are drawn in order and never reused, which keeps the ranges disjoint. + """ + + max_ngram_size: int + layer_ids: tuple[int, ...] + num_embeddings: tuple[int, ...] # table rows, per engram layer + primes: tuple[tuple[tuple[int, ...], ...], ...] # [layer][n-gram size][head] bucket modulus + n_heads: int + head_dim: int + + @classmethod + def from_args(cls, args) -> "EngramLayout | None": + layer_ids = tuple(args.engram_layer_ids) + if not layer_ids: + return None + max_ngram_size, n_heads = args.engram_max_ngram_size, args.engram_n_heads + primes, seen = [], set() + for _ in layer_ids: + per_ngram = [] + for _ in range(max_ngram_size - 1): + sizes, current = [], args.engram_vocab_size - 1 + for _ in range(n_heads): + current = find_next_prime(current, seen) + seen.add(current) + sizes.append(current) + per_ngram.append(tuple(sizes)) + primes.append(tuple(per_ngram)) + return cls( + max_ngram_size=max_ngram_size, + layer_ids=layer_ids, + num_embeddings=tuple(args.engram_num_embeddings), + primes=tuple(primes), + n_heads=n_heads, + head_dim=args.engram_head_dim, + ) + + +class NgramHashState(nn.Module): + """Maps each position to the hash ids of the n-grams ending there. + + Ids go through the compressed table, then each position is hashed with the `max_ngram_size - 1` + tokens before it. Look-back stops at the start of the sequence and at any dead token (an image + span, cached as DEAD), so an n-gram never spans one. The cache carries all of this across the + prefill/decode split. + """ + + DEAD = -1 + + def __init__(self, args, layout: EngramLayout, tokenizer): + super().__init__() + self.layout = layout + # every hash multiplier derives from the compressed vocab size, so a mismatch there would + # silently rehash the whole table + token_map, vocab_size = build_compressed_token_map(tokenizer) + assert vocab_size == args.engram_compressed_vocab_size, (vocab_size, args.engram_compressed_vocab_size) + self.pad_id = token_map[args.engram_pad_id] + flat = [[p for per_ngram in layer for p in per_ngram] for layer in layout.primes] + offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat] + multipliers = compute_hash_multipliers(layout.layer_ids, layout.max_ngram_size, vocab_size) + self.register_buffer("primes", torch.tensor(layout.primes), persistent=False) + self.register_buffer("offsets", torch.tensor(np.array(offsets)), persistent=False) + self.register_buffer("multipliers", multipliers, persistent=False) + self.register_buffer("token_map", torch.tensor(token_map), persistent=False) + self.register_buffer( + "cache", torch.empty(args.max_batch_size, args.max_seq_len, dtype=torch.int64), persistent=False + ) + + @torch.inference_mode() + def forward(self, input_ids: torch.Tensor, start_pos: int, token_mask: torch.Tensor | None = None) -> torch.Tensor: + """token_mask: [B, L], False for tokens that take no part in an n-gram (image spans). + Returns the hash ids, shaped [B, L, n_engram_layers, n_hash_cols].""" + batch, seqlen = input_ids.shape + compressed = self.token_map[input_ids] + if token_mask is not None: + compressed = torch.where(token_mask, compressed, self.DEAD) + self.cache[:batch, start_pos : start_pos + seqlen] = compressed + + positions = torch.arange(start_pos, start_pos + seqlen, device=input_ids.device).expand(batch, seqlen) + tokens, blocked = [], torch.zeros_like(positions, dtype=torch.bool) + for shift in range(self.layout.max_ngram_size): + source = self.cache[:batch].gather(1, (positions - shift).clamp_min(0)) + blocked = blocked | (positions < shift) | (source == self.DEAD) + tokens.append(torch.where(blocked, self.pad_id, source)) + tokens = torch.stack(tokens, dim=-1) # [B, L, max_ngram_size] + + # XOR the multiplied ids together one lookback at a time, so the running value after step i + # is the hash of the (i+1)-gram; each lands in its own prime-sized bucket range + products = tokens.unsqueeze(2) * self.multipliers # [B, L, n_engram_layers, max_ngram_size] + rolling, hashes = products[..., 0], [] + for i in range(1, self.layout.max_ngram_size): + rolling = torch.bitwise_xor(rolling, products[..., i]) + hashes.append(rolling.unsqueeze(-1) % self.primes[:, i - 1]) + return torch.cat(hashes, dim=-1) + self.offsets diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/image_processor.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/image_processor.py new file mode 100644 index 000000000000..2a0dce43f3db --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/image_processor.py @@ -0,0 +1,179 @@ +# SPDX-FileCopyrightText: © 2023 DeepSeek +# SPDX-License-Identifier: MIT +# Vendored from huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash, revision +# dba1be0a40aa45a94ad051997016db3960a90277, file inference/image_processor.py (MIT License). +# Deviations from upstream are listed in README.md next to this file. + +"""Image preprocessing. + +An image becomes a `n_vit_h x n_vit_w` patch grid for the ViT and a `n_llm_h x n_llm_w` token grid +after the 3x3 aligner downsample, which the LLM sees as + + [IMAGE_START] + ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + [IMAGE_END] + +Every one of those positions carries `image_token_id` in `input_ids`; only the token type tells them +apart. The IMAGE slots are filled with aligner rows in reading order. +""" + +import base64 +import io +import math +from dataclasses import dataclass +from urllib.request import urlopen + +import numpy as np +import torch +from PIL import Image, ImageOps + +TEXT = -1 +IMAGE_START, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(4) + + +@dataclass +class ImageInput: + start: int + patches: torch.Tensor + n_vit_h: int + n_vit_w: int + types: torch.Tensor + + +def num_image_tokens(n_llm_h: int, n_llm_w: int) -> int: + return n_llm_h * (n_llm_w + 1) + 2 + + +def llm_grid(best_height: int, best_width: int, patch_size: int, downsample_ratio: int): + """Token grid the aligner produces from a patch grid of this pixel size.""" + return math.ceil((best_height // patch_size) / downsample_ratio), math.ceil( + (best_width // patch_size) / downsample_ratio + ) + + +def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token): + """Largest aspect-preserving pixel size whose token grid still fits in max_n_token.""" + r = height / width + max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5 + max_h_float = max_w_float * r + cell = patch_size * downsample_ratio + if max_w_float < 1.0: # very tall: collapse to a single column + return (max_n_token - 2) // 2 * cell, cell + if max_h_float < 1.0: # very wide: collapse to a single row + return cell, (max_n_token - 3) * cell + beta = min(math.floor(max_w_float) * cell / width, math.floor(max_h_float) * cell / height) + return math.floor(height * beta / patch_size) * patch_size, math.floor(width * beta / patch_size) * patch_size + + +def safe_resize(height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token): + """Shrink the pixel size until the image costs at most max_n_token LLM tokens.""" + n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio) + if num_image_tokens(n_llm_h, n_llm_w) > max_n_token: + best_height, best_width = solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token) + n_llm_h, n_llm_w = llm_grid(best_height, best_width, patch_size, downsample_ratio) + assert num_image_tokens(n_llm_h, n_llm_w) <= max_n_token + return n_llm_h, n_llm_w, best_height, best_width + + +def load_image_bytes(record) -> bytes: + """Load image bytes from raw/base64 data, an Anthropic source, URL, or path.""" + data = record.get("data") + if isinstance(data, bytes): + return data + if isinstance(data, str): + return base64.b64decode(data) + + source = record.get("source") + if isinstance(source, dict): + if source.get("data") is not None: + return base64.b64decode(source["data"]) + if source.get("url"): + return load_image_bytes({"url": source["url"]}) + + url = record.get("url") + if isinstance(url, str) and url: + if url.startswith("data:"): + header, _, payload = url.partition(",") + if ";base64" not in header: + raise ValueError(f"Unsupported data URL encoding: {header}") + return base64.b64decode(payload) + if url.startswith(("http://", "https://")): + with urlopen(url, timeout=30) as response: + return response.read() + with open(url, "rb") as file: + return file.read() + + raise ValueError(f"Cannot load image from record: {list(record.keys())}") + + +def plan_image_grid(width: int, height: int, args): + """Resize plan for an image of the given original size; a pure function of its arguments.""" + p = args.vision_patch_size + if args.vision_max_wh_ratio is not None and width > height * args.vision_max_wh_ratio: + width = height * args.vision_max_wh_ratio + if 0 < width * height < args.vision_min_pixels: + ratio = (args.vision_min_pixels / (width * height)) ** 0.5 + width = int(width * ratio) + height = int(height * ratio) + best_width = math.ceil(width / p) * p + best_height = math.ceil(height / p) * p + return safe_resize(height, width, best_height, best_width, p, args.vision_downsample_ratio, args.vision_max_n_token) + + +def load_image(record, args): + """Load and transform one image record into ViT patches.""" + p = args.vision_patch_size + with Image.open(io.BytesIO(load_image_bytes(record))) as source: + image = source.convert("RGB") + n_llm_h, n_llm_w, best_height, best_width = plan_image_grid(image.width, image.height, args) + n_vit_h, n_vit_w = best_height // p, best_width // p + if args.vision_max_wh_ratio is not None and image.width >= args.vision_max_wh_ratio * image.height: + image = image.resize((best_width, best_height)) + else: + image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127)) + x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255 + x = ((x - 0.5) / 0.5).to(torch.bfloat16) + patches = x.reshape(3, n_vit_h, p, n_vit_w, p).permute(1, 3, 0, 2, 4).reshape(n_vit_h * n_vit_w, 3, p, p) + return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w + + +def image_token_types(n_llm_h: int, n_llm_w: int) -> torch.Tensor: + """Default layout: the aligner grid in reading order, one IMAGE_NEW_LINE per row.""" + types = [IMAGE_START] + types += ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + types.append(IMAGE_END) + return torch.tensor(types, dtype=torch.int64) + + +def prepare_vl_inputs(prompt, images, tokenizer, args): + """Tokenize `prompt`, expanding each image placeholder token into its image span. + + Returns (tokens, token_types, image_inputs). Image-span positions carry `args.image_token_id` in + `tokens` and are distinguished only by `token_types` (TEXT elsewhere). `image_inputs` is None when + the prompt has no images.""" + from encoding import IMAGE_PLACEHOLDER + + # The placeholder is spelled differently across tokenizer revisions, so the id comes from the + # config; only cross-check it when this tokenizer does know the training-time spelling. + image_token_id = args.image_token_id + placeholder_id = tokenizer.convert_tokens_to_ids(IMAGE_PLACEHOLDER) + if placeholder_id is not None and placeholder_id != tokenizer.unk_token_id: + assert placeholder_id == image_token_id, (placeholder_id, image_token_id) + prompt_tokens = tokenizer.encode(prompt) + num_placeholders = sum(token == image_token_id for token in prompt_tokens) + if num_placeholders != len(images): + raise ValueError(f"Found {num_placeholders} image tokens but got {len(images)} images") + if num_placeholders and not args.vision_enabled: + raise ValueError("The model config has no vision tower (vision_n_layers == 0) but the prompt contains images") + + tokens, token_types, image_inputs = [], [], [] + image_iter = iter(images) + for tok in prompt_tokens: + if tok != image_token_id: + tokens.append(tok) + token_types.append(TEXT) + continue + patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(next(image_iter), args) + types = image_token_types(n_llm_h, n_llm_w) + image_inputs.append(ImageInput(len(tokens), patches, n_vit_h, n_vit_w, types)) + tokens += [image_token_id] * types.numel() + token_types += types.tolist() + return tokens, token_types, image_inputs or None diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/kernel_cpu.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/kernel_cpu.py new file mode 100644 index 000000000000..19507c9f823a --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/kernel_cpu.py @@ -0,0 +1,271 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Torch (CPU) ports of the DeepSeek-V4.1-Flash tilelang kernels. + +Drop-in replacement for upstream ``inference/kernel.py`` (HF ``deepseek-ai/DeepSeek-V4.1-Flash`` at +revision dba1be0a40aa45a94ad051997016db3960a90277): same function names, signatures, return types and +numerics. Each function documents the kernel lines it mirrors (``K41:`` = upstream kernel.py). + +Exactness: + +* ``act_quant`` / ``fp4_act_quant`` are elementwise over fixed groups, so they are bit-exact ports + (scale rounding reproduces ``fast_round_scale``'s IEEE-754 bit manipulation; FP8/FP4 casts are + round-to-nearest-even with saturation, like the CUDA ``satfinite`` conversions). +* ``hc_split_sinkhorn`` performs the same fp32 operations in the same order; only ``exp``/``sigmoid`` + transcendental implementations may differ from CUDA (last-ulp fp32). +* ``fp8_gemm`` / ``fp4_gemm`` accumulate each K block in fp32 (block products are exact because + FP8 x FP8 and FP4 x FP8 products fit in fp32) and add the scaled block partials sequentially in K + order, as the kernel does. Only the summation order inside one K block differs from the tensor core, + which can change the fp32 result by a few ulp, i.e. at most 1 bf16 ulp of the output, rarely. +* ``sparse_attn`` reproduces the online softmax over blocks of 64 indices, including the bf16 rounding + of the probabilities relative to the running (not final) max; remaining differences come from + GEMM summation order and ``exp`` implementation, expected within ~1 bf16 ulp of the output. +""" + +from typing import Optional, Tuple + +import torch + +FP8_MAX = 448.0 +FP4_MAX = 6.0 +# The kernels multiply by the reciprocal as a float32 constant (K41:47,138: fp8_max_inv / fp4_max_inv). +_FP8_MAX_INV = torch.tensor(1.0 / FP8_MAX, dtype=torch.float32) +_FP4_MAX_INV = torch.tensor(1.0 / FP4_MAX, dtype=torch.float32) + +# E2M1 magnitudes by 3-bit code, and the sign-extended 4-bit table (same as upstream convert.py FP4_TABLE). +_E2M1_GRID = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32) +FP4_TABLE = torch.cat([_E2M1_GRID, -_E2M1_GRID]) +# Round-to-nearest-even boundaries between consecutive codes; at an exact tie the result goes up only +# when the upper code is even, i.e. for boundaries 1, 3, 5 (0.75 -> 1.0, 1.75 -> 2.0, 3.5 -> 4.0). +_E2M1_MIDS = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0], dtype=torch.float32) +_E2M1_TIE_UP = torch.tensor([False, True, False, True, False, True, False]) + +SPARSE_ATTN_BLOCK = 64 # K41:328 `block` + + +def fast_round_scale(amax: torch.Tensor, max_inv: torch.Tensor) -> torch.Tensor: + """2^ceil(log2(amax * max_inv)) computed exactly like K41:22-38 (exponent bits + mantissa != 0).""" + x = (amax.float() * max_inv).contiguous() + bits = x.view(torch.int32) + exp = (bits >> 23) & 0xFF + man = bits & ((1 << 23) - 1) + log2_ceil = exp - 127 + (man != 0).to(torch.int32) + return ((log2_ceil + 127) << 23).view(torch.float32) + + +def _to_e4m3_satfinite(x: torch.Tensor) -> torch.Tensor: + """fp32 -> float8_e4m3fn, round-to-nearest-even, saturating at +-448 (torch alone maps overflow to NaN).""" + return x.clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + + +def _e2m1_codes(x: torch.Tensor) -> torch.Tensor: + """fp32 in [-6, 6] -> 4-bit E2M1 codes (bit 3 = sign), round-to-nearest-even.""" + mag = x.abs().unsqueeze(-1) + code = (mag > _E2M1_MIDS).sum(-1) + ((mag == _E2M1_MIDS) & _E2M1_TIE_UP).sum(-1) + return code.to(torch.uint8) | (torch.signbit(x).to(torch.uint8) << 3) + + +def _e2m1_round(x: torch.Tensor) -> torch.Tensor: + """fp32 in [-6, 6] -> nearest E2M1 value as fp32 (sign of zero preserved).""" + return FP4_TABLE[_e2m1_codes(x).long()] + + +def unpack_fp4(x: torch.Tensor) -> torch.Tensor: + """float4_e2m1fn_x2 [..., K//2] -> fp32 [..., K]; the low nibble holds the even element.""" + b = x.view(torch.uint8) + vals = torch.stack([FP4_TABLE[(b & 0x0F).long()], FP4_TABLE[(b >> 4).long()]], dim=-1) + return vals.flatten(-2) + + +def pack_fp4(codes: torch.Tensor) -> torch.Tensor: + """4-bit codes uint8 [..., K] -> float4_e2m1fn_x2 [..., K//2] (even element in the low nibble).""" + return (codes[..., 0::2] | (codes[..., 1::2] << 4)).contiguous().view(torch.float4_e2m1fn_x2) + + +def act_quant( + x: torch.Tensor, + block_size: int = 128, + scale_fmt: Optional[str] = None, + scale_dtype: torch.dtype = torch.float32, + inplace: bool = False, +) -> torch.Tensor: + """Block-wise FP8 quantization (K41:41-113). inplace=True does fused quant+dequant back to BF16. + When scale_fmt is set, scales are rounded to power-of-2 (MXFP).""" + N = x.size(-1) + assert N % block_size == 0 + assert x.dtype == torch.bfloat16, "act_quant_kernel is compiled for bf16 input (K41:41 in_dtype=BF16)" + xg = x.float().unflatten(-1, (-1, block_size)) + amax = xg.abs().amax(dim=-1).clamp_min(1e-4) # K41:76-77 + if scale_fmt is not None: + s = fast_round_scale(amax, _FP8_MAX_INV) + else: + s = amax * _FP8_MAX_INV + q = _to_e4m3_satfinite(xg / s.unsqueeze(-1)) # K41:81-89: clamp to +-448, cast to e4m3 + if inplace: + y = (q.float() * s.unsqueeze(-1)).flatten(-2).to(x.dtype) + x.copy_(y) + return x + return q.flatten(-2), s.to(scale_dtype) + + +def fp4_act_quant( + x: torch.Tensor, + block_size: int = 32, + inplace: bool = False, + scale_dtype: torch.dtype = torch.float8_e8m0fnu, +) -> torch.Tensor: + """FP4 with E8M0 scales for the indexer or E4M3 scales for compressed KV (K41:116-199). + inplace=True writes the dequantized values back to x.""" + assert scale_dtype in (torch.float8_e8m0fnu, torch.float8_e4m3fn) + N = x.size(-1) + assert N % block_size == 0 + assert x.dtype == torch.bfloat16, "fp4_quant_kernel is compiled for bf16 input (K41:116 in_dtype=BF16)" + xg = x.float().unflatten(-1, (-1, block_size)) + amax = xg.abs().amax(dim=-1) + if scale_dtype == torch.float8_e4m3fn: + # K41:158-161: keep even an all-zero group's scale nonzero; scale is amax/6 rounded to E4M3 + amax = amax.clamp_min(6 * (2**-9)) + s = _to_e4m3_satfinite(amax / FP4_MAX).float() + else: + amax = amax.clamp_min(6 * (2**-126)) # K41:163-164 + s = fast_round_scale(amax, _FP4_MAX_INV) + v = (xg / s.unsqueeze(-1)).clamp(-FP4_MAX, FP4_MAX) + if inplace: + y = (_e2m1_round(v) * s.unsqueeze(-1)).flatten(-2).to(x.dtype) # K41:165-170 + x.copy_(y) + return x + return pack_fp4(_e2m1_codes(v).flatten(-2)), s.to(scale_dtype) + + +def _block_scaled_gemm(a: torch.Tensor, a_s: torch.Tensor, b: torch.Tensor, b_s: torch.Tensor, block_k: int): + """sum_k (A_k @ B_k^T) * a_s[:, k] * b_s[:, k] in fp32. + + a: [M, K] fp32 values, a_s: [M, K//block_k] fp32, b: [N, K] fp32 values, b_s: [N, K//block_k] fp32. + Scales are powers of two (ue8m0) or exact products, so scaling the operands before one fp32 matmul is + exact per element; only the fp32 summation order differs from the kernel's per-block accumulation + (same tolerance class as the tensor-core order, see module docstring). A per-block Python loop costs + minutes per layer at real dims. + """ + a_scaled = a * a_s.repeat_interleave(block_k, dim=1) + b_scaled = b * b_s.repeat_interleave(block_k, dim=1) + return a_scaled @ b_scaled.T + + +def fp8_gemm( + a: torch.Tensor, + a_s: torch.Tensor, + b: torch.Tensor, + b_s: torch.Tensor, + scale_dtype: torch.dtype = torch.float32, + block_size: int = 128, +) -> torch.Tensor: + """C[M,N] = A[M,K] @ B[N,K]^T with per-block FP8 scaling (K41:202-293).""" + assert a.is_contiguous() and b.is_contiguous(), "Input tensors must be contiguous" + assert a_s.is_contiguous() and b_s.is_contiguous(), "Scaling factor tensors must be contiguous" + assert block_size in (32, 128) + assert a.dtype == torch.float8_e4m3fn and b.dtype == torch.float8_e4m3fn + K = a.size(-1) + M = a.numel() // K + N = b.size(0) + assert K % block_size == 0 + assert a_s.numel() == M * (K // block_size) + assert b_s.shape == ( + (N + block_size - 1) // block_size, + K // block_size, + ) + # weight scales are per (N block, K block): expand to per output row + b_s_rows = b_s.float().repeat_interleave(block_size, dim=0)[:N] + c = _block_scaled_gemm(a.view(M, K).float(), a_s.view(M, -1).float(), b.float(), b_s_rows, block_size) + return c.to(torch.get_default_dtype()).view(*a.size()[:-1], N) + + +def fp4_gemm( + a: torch.Tensor, + a_s: torch.Tensor, + b: torch.Tensor, + b_s: torch.Tensor, + scale_dtype: torch.dtype = torch.float32, + act_block_size: int = 128, +) -> torch.Tensor: + """C[M,N] = A_fp8[M,K] @ B_fp4[N,K]^T (K41:461-591). + A has per-32 or per-128 activation scale; B has per-32 E8M0 weight scale. + B is stored as [N, K//2] in float4_e2m1fn_x2 (2 FP4 values per byte, packed along K).""" + assert a.is_contiguous() and b.is_contiguous(), "Input tensors must be contiguous" + assert a_s.is_contiguous() and b_s.is_contiguous(), "Scaling factor tensors must be contiguous" + assert a.dtype == torch.float8_e4m3fn and b.dtype == torch.float4_e2m1fn_x2 + K = a.size(-1) + M = a.numel() // K + N = b.size(0) + assert act_block_size in (32, 128) + assert K % act_block_size == 0 + assert a_s.numel() == M * (K // act_block_size) + assert b_s.shape == (N, K // 32) + # the kernel walks K in blocks of 32 (the weight group); the act scale index is k // (act_block/32) + a_s_32 = a_s.view(M, -1).float().repeat_interleave(act_block_size // 32, dim=1) + c = _block_scaled_gemm(a.view(M, K).float(), a_s_32, unpack_fp4(b), b_s.float(), 32) + return c.to(torch.get_default_dtype()).view(*a.size()[:-1], N) + + +def sparse_attn( + q: torch.Tensor, kv: torch.Tensor, attn_sink: torch.Tensor, topk_idxs: torch.Tensor, softmax_scale: float +) -> torch.Tensor: + """Sparse single-KV-head attention over gathered indices with an attention sink (K41:296-403). + + q [b, m, h, d] bf16, kv [b, n, d] bf16 (K = V), attn_sink [h] fp32, topk_idxs [b, m, topk] int32. + Index -1 contributes nothing; a row with no valid index yields zeros. Output bf16 [b, m, h, d]. + """ + b, m, h, d = q.size() + topk = topk_idxs.size(-1) + scale = torch.tensor(softmax_scale, dtype=torch.float32) + qf = q.float() + kvf = kv.float() + acc_o = torch.zeros(b, m, h, d, dtype=torch.float32) + sum_exp = torch.zeros(b, m, h, dtype=torch.float32) + # finite lower bound (K41:352-355): all -1 rows give exp(-inf - (-1e30)) = 0, not NaN + scores_max = torch.full((b, m, h), -1e30, dtype=torch.float32) + batch = torch.arange(b).view(b, 1, 1) + for t in range(0, topk, SPARSE_ATTN_BLOCK): + idxs = topk_idxs[:, :, t : t + SPARSE_ATTN_BLOCK].long() + valid = idxs != -1 + kv_blk = kvf[batch, idxs.clamp_min(0)] * valid.unsqueeze(-1) # [b, m, blk, d], invalid rows zeroed + acc_s = torch.einsum("bmhd,bmkd->bmhk", qf, kv_blk) + acc_s = torch.where(valid.unsqueeze(2), acc_s, -torch.inf) * scale # K41:365-368 + scores_max_prev = scores_max + scores_max = torch.maximum(scores_max_prev, acc_s.amax(dim=-1)) + scores_scale = torch.exp(scores_max_prev - scores_max) + p = torch.exp(acc_s - scores_max.unsqueeze(-1)) + sum_exp = sum_exp * scores_scale + p.sum(dim=-1) # fp32 probabilities in the denominator + p_bf16 = p.to(torch.bfloat16) # K41:377: probabilities cast to bf16 before PV + acc_o = acc_o * scores_scale.unsqueeze(-1) + torch.einsum("bmhk,bmkd->bmhd", p_bf16.float(), kv_blk) + sum_exp = sum_exp + torch.exp(attn_sink.float() - scores_max) # K41:382-383: sink not scaled + return (acc_o / sum_exp.unsqueeze(-1)).to(torch.bfloat16) + + +def hc_split_sinkhorn( + mixes: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + hc_mult: int = 4, + sinkhorn_iters: int = 20, + eps: float = 1e-6, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Split hc mixes into pre / post / Sinkhorn-normalized comb (K41:406-458), all fp32.""" + hc = hc_mult + m = mixes.float() + hc_scale = hc_scale.float() + hc_base = hc_base.float() + pre = torch.sigmoid(m[..., :hc] * hc_scale[0] + hc_base[:hc]) + eps + post = 2 * torch.sigmoid(m[..., hc : 2 * hc] * hc_scale[1] + hc_base[hc : 2 * hc]) + comb = m[..., 2 * hc :] * hc_scale[2] + hc_base[2 * hc :] + comb = comb.unflatten(-1, (hc, hc)) + # comb = comb.softmax(-1) + eps + comb = torch.exp(comb - comb.amax(dim=-1, keepdim=True)) + comb = comb / comb.sum(dim=-1, keepdim=True) + eps + # comb = comb / (comb.sum(-2) + eps) + comb = comb / (comb.sum(dim=-2, keepdim=True) + eps) + for _ in range(sinkhorn_iters - 1): + comb = comb / (comb.sum(dim=-1, keepdim=True) + eps) + comb = comb / (comb.sum(dim=-2, keepdim=True) + eps) + return pre, post, comb diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/model.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/model.py new file mode 100644 index 000000000000..94a0dff4c447 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/model.py @@ -0,0 +1,1318 @@ +# SPDX-FileCopyrightText: © 2023 DeepSeek +# SPDX-License-Identifier: MIT +# Vendored from huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash, revision +# dba1be0a40aa45a94ad051997016db3960a90277, file inference/model.py (MIT License). +# Deviations from upstream are listed in README.md next to this file. +# Formatter directives keep the vendored code byte-identical to upstream: +# isort: skip_file +# fmt: off + +import math +from contextlib import contextmanager +from dataclasses import dataclass +from functools import lru_cache +from typing import Literal + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from .engram import EngramLayout, NgramHashState +from .image_processor import IMAGE, IMAGE_END, IMAGE_NEW_LINE, IMAGE_START +from .kernel_cpu import ( + act_quant, + fp4_act_quant, + fp4_gemm, + fp8_gemm, + hc_split_sinkhorn, + sparse_attn, +) +from torch import nn +from .vision import Aligner, ViT + +# Set once by Transformer.__init__; one model per process, so layers just read them. +world_size = 1 +rank = 0 +default_dtype = torch.float8_e4m3fn # storage dtype for Linear weights, from ModelArgs.dtype +fp8_block_size = 32 # one fp8 scale per 32x32 weight block / 32 activations +fp4_block_size = 32 # one fp4 scale per 32 elements along K +scale_fmt = "ue8m0" +scale_dtype = torch.float8_e8m0fnu + + +@contextmanager +def set_dtype(dtype): + """Temporarily override torch's default dtype, restoring it even if the body raises.""" + prev = torch.get_default_dtype() + torch.set_default_dtype(dtype) + try: + yield + finally: + torch.set_default_dtype(prev) + + +@dataclass +class ModelArgs: + """Field names are exactly the config JSON keys. The defaults are a small model that + `python model.py` can run, not the released shapes -- though the scale-independent + values (norm_eps, score_func, hc_*, engram_*) do match it.""" + + # runtime limits rather than model shape: they size the KV caches + max_batch_size: int = 4 + max_seq_len: int = 4096 + temperature: float = 1 + dtype: Literal["bf16", "fp8"] = "fp8" + expert_dtype: Literal["fp4"] | None = "fp4" + vocab_size: int = 129280 + dim: int = 1024 + moe_inter_dim: int = 1024 + n_layers: int = 5 + n_mtp_layers: int = 1 # extra draft layers appended after the backbone, indices n_layers.. + n_heads: int = 16 + # moe + n_routed_experts: int = 8 + n_shared_experts: int = 1 + n_activated_experts: int = 2 + score_func: Literal["softmax", "sigmoid", "sqrtsoftplus"] = "sqrtsoftplus" + gate_temp: float = 1.0 + norm_topk_prob: bool = True + route_scale: float = 1.0 + swiglu_limit: float = 0.0 + # attention: latent q/kv projections, plus a LoRA-factorised output projection over o_groups + q_lora_rank: int = 256 + head_dim: int = 128 + rope_head_dim: int = 32 + norm_eps: float = 1e-20 + o_groups: int = 8 + o_lora_rank: int = 256 + # sparse attention: every layer attends over a sliding window, and may add compressed KV on top + window_size: int = 128 + # one entry per layer, MTP layers included: 0 = sliding window only, r = KV compressed r-to-1 + compress_ratios: tuple[int, ...] = (0, 2, 2, 1, 1, 0) + # layers sharing a ratio also share one compressed KV and one indexer, produced by the first + kv_source_layers: tuple[int, ...] = (1, 3) + index_source_layers: tuple[int, ...] = (1, 3) + # rope, with YaRN extrapolation when original_seq_len > 0. Compressed KV rotates at its own + # theta because one latent stands for compress_ratio tokens, so its positions are further apart. + compress_rope_theta: float = 40000.0 + original_seq_len: int = 0 + rope_theta: float = 10000.0 + rope_factor: float = 40 + beta_fast: int = 32 + beta_slow: int = 1 + # the indexer: a small extra attention that scores compressed positions, so each query can keep + # just `index_topk` of them. Names match DeepSeek-V3.2-Exp, where this mechanism first appeared. + index_n_heads: int = 16 + index_head_dim: int = 64 + index_topk: int = 64 + # candidate pre-filtering: candidate_source_layer < 0 turns it off and the other two are unused + candidate_source_layer: int = -1 + candidate_topk_blocks: int = 0 + candidate_block_size: int = 0 + # hyper-connections: the residual stream is carried as hc_mult parallel copies + hc_mult: int = 4 + hc_sinkhorn_iters: int = 20 + hc_eps: float = 1e-6 + # engram: n-gram hash lookups added into the residual stream at a few layers + engram_layer_ids: tuple[int, ...] = () + engram_num_embeddings: tuple[int, ...] = () # unpadded table rows; each rank allocates ceil(rows / world_size) + engram_max_ngram_size: int = 1 + engram_vocab_size: int = 0 # bucket size each (n-gram size, head) starts searching primes from + engram_n_heads: int = 0 + engram_head_dim: int = 0 + engram_pad_id: int = 2 # token that fills n-gram slots with no history; matches training + # size of the compressed tokenizer vocab; every hash multiplier is derived from it + engram_compressed_vocab_size: int = 0 + # vision (VL); vision_n_layers == 0 disables the vision path + vision_n_layers: int = 0 + vision_dim: int = 1024 + vision_n_heads: int = 16 + vision_inter_dim: int = 2816 + vision_patch_size: int = 14 + vision_rope_theta: float = 10000.0 + vision_downsample_ratio: int = 3 + vision_max_n_token: int = 1024 + vision_min_pixels: int = 544 * 544 + vision_max_wh_ratio: int | None = None + # raw id of <|deepseek_image|>; every position of an image span carries this id in input_ids + image_token_id: int = 129264 + # dspark draft head. Only the forward pass is implemented here -- nothing calls forward_spec, + # so these are read but the speculative-decoding loop itself is out of scope for this repo. + dspark_block_size: int = 0 + dspark_noise_token_id: int = 0 + dspark_target_layer_ids: tuple[int, ...] = () + dspark_markov_rank: int = 256 + dspark_n_routed_experts: int = 0 + dspark_n_activated_experts: int = 0 + + @property + def vision_enabled(self) -> bool: + return self.vision_n_layers > 0 + + def get_moe_config(self, layer_id: int) -> tuple[int, int]: + """Return the routed/activated expert counts for a given layer.""" + if layer_id < self.n_layers: + return self.n_routed_experts, self.n_activated_experts + return ( + self.dspark_n_routed_experts or self.n_routed_experts, + self.dspark_n_activated_experts or self.n_activated_experts, + ) + + +class ParallelEmbedding(nn.Module): + """Embedding sharded along the vocab dimension. Each rank holds vocab_size // world_size rows. + Out-of-range indices are zero-masked before all_reduce to combine partial embeddings.""" + + def __init__(self, vocab_size: int, dim: int): + super().__init__() + self.vocab_size = vocab_size + self.dim = dim + assert vocab_size % world_size == 0, ( + f"Vocabulary size must be divisible by world size (world_size={world_size})" + ) + self.part_vocab_size = vocab_size // world_size + self.vocab_start_idx = rank * self.part_vocab_size + self.vocab_end_idx = self.vocab_start_idx + self.part_vocab_size + self.weight = nn.Parameter(torch.empty(self.part_vocab_size, self.dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if world_size > 1: + # ids off this rank read row 0 then get zeroed, so the all_reduce sums one real row + mask = (x < self.vocab_start_idx) | (x >= self.vocab_end_idx) + x = x - self.vocab_start_idx + x[mask] = 0 + y = F.embedding(x, self.weight) + if world_size > 1: + y[mask] = 0 + dist.all_reduce(y) + return y + + +def linear(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: + """Pick a GEMM from the weight dtype. Quantized weights need a quantized activation, and both + fp4 and fp8 weights take an fp8 one -- for fp4 the kernel handles the mixed precision.""" + assert bias is None + + if weight.dtype == torch.float4_e2m1fn_x2: + x, s = act_quant(x, fp8_block_size, scale_fmt, scale_dtype) + return fp4_gemm( + x, + s, + weight, + weight.scale, + scale_dtype, + act_block_size=fp8_block_size, + ) + elif weight.dtype == torch.float8_e4m3fn: + x, s = act_quant(x, fp8_block_size, scale_fmt, scale_dtype) + return fp8_gemm( + x, + s, + weight, + weight.scale, + scale_dtype, + block_size=fp8_block_size, + ) + else: + return F.linear(x, weight) + + +class Linear(nn.Module): + """bf16, fp8 or fp4 weights. Quantized ones get a `scale`, also attached to `.weight` so that + `linear()` can reach it.""" + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype=None): + super().__init__() + self.in_features = in_features + self.out_features = out_features + dtype = dtype or default_dtype + if dtype == torch.float4_e2m1fn_x2: + # two values per byte: [out, in] logically, [out, in//2] stored + self.weight = nn.Parameter(torch.empty(out_features, in_features // 2, dtype=torch.float4_e2m1fn_x2)) + self.weight.scale = self.scale = nn.Parameter( + torch.empty(out_features, in_features // fp4_block_size, dtype=torch.float8_e8m0fnu) + ) + elif dtype == torch.float8_e4m3fn: + self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype)) + self.weight.scale = self.scale = nn.Parameter( + torch.empty( + (out_features + fp8_block_size - 1) // fp8_block_size, + (in_features + fp8_block_size - 1) // fp8_block_size, + dtype=torch.float8_e8m0fnu, + ) + ) + else: + self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype)) + self.register_parameter("scale", None) + if bias: + self.bias = nn.Parameter(torch.empty(out_features)) + else: + self.register_parameter("bias", None) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return linear(x, self.weight, self.bias) + + +class ColumnParallelLinear(Linear): + """Splits the output dim across ranks; each rank's slice of the output is already complete.""" + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype=None): + assert out_features % world_size == 0, ( + f"Output features must be divisible by world size (world_size={world_size})" + ) + self.part_out_features = out_features // world_size + super().__init__(in_features, self.part_out_features, bias, dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return linear(x, self.weight, self.bias) + + +class RowParallelLinear(Linear): + """Splits the reduction dim, so each rank holds a partial sum: hence the fp32 all_reduce, with + the bias added only after it.""" + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype=None): + assert in_features % world_size == 0, ( + f"Input features must be divisible by world size (world_size={world_size})" + ) + self.part_in_features = in_features // world_size + super().__init__(self.part_in_features, out_features, bias, dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = linear(x, self.weight, None) + if world_size > 1: + y = y.float() + dist.all_reduce(y) + if self.bias is not None: + y += self.bias + return y.type_as(x) + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x: torch.Tensor): + dtype = x.dtype + x = x.float() + var = x.square().mean(-1, keepdim=True) + x = x * torch.rsqrt(var + self.eps) + return (self.weight * x).to(dtype) + + +class ParallelEngramEmbedding(nn.Module): + """The n-gram hash table, sharded over its rows. Stays fp8: rows are dequantized on lookup.""" + + def __init__(self, num_embeddings: int, dim: int): + super().__init__() + self.num_embeddings = num_embeddings + self.dim = dim + self.part_num_embeddings = (num_embeddings + world_size - 1) // world_size + self.vocab_start_idx = rank * self.part_num_embeddings + self.vocab_end_idx = self.vocab_start_idx + self.part_num_embeddings + + self.block_size = fp8_block_size + # the table stays fp8 as stored: rows are dequantized with `scale` on lookup + self.weight = nn.Parameter(torch.empty(self.part_num_embeddings, dim, dtype=torch.float8_e4m3fn)) + self.scale = nn.Parameter(torch.empty(self.part_num_embeddings, dim // self.block_size, dtype=scale_dtype)) + + def forward(self, indices: torch.Tensor) -> torch.Tensor: + mask = (indices < self.vocab_start_idx) | (indices >= self.vocab_end_idx) + local_indices = indices - self.vocab_start_idx + local_indices = local_indices.masked_fill(mask, 0) + + values = F.embedding(local_indices, self.weight) + scales = F.embedding(local_indices, self.scale) + values = values.float().unflatten(-1, (-1, self.block_size)) * scales.float().unsqueeze(-1) + values = values.flatten(-2).to(torch.bfloat16) + values = values.masked_fill(mask.unsqueeze(-1), 0) + + if world_size > 1: + dist.all_reduce(values) + return values + + +class Engram(nn.Module): + """Writes an n-gram lookup into the residual stream, gated by how well it matches that stream. + + The hash ids fetch `n_hash_cols` rows; `wkv` turns them into one key per hc copy plus a shared + value. The gate is a normalized dot product of stream against key. + """ + + def __init__(self, args: ModelArgs, layer_id: int, layout: EngramLayout): + super().__init__() + self.layer_id = layer_id + self.layer_hash_index = layout.layer_ids.index(layer_id) + self.dim = args.dim + self.hc_mult = args.hc_mult + self.clamp_value = 1e-6 + + self.embed = ParallelEngramEmbedding(layout.num_embeddings[self.layer_hash_index], layout.head_dim) + n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads + self.wkv = Linear(n_hash_cols * layout.head_dim, args.dim * (args.hc_mult + 1)) + self.eps = args.norm_eps + self.q_weight = nn.Parameter(torch.ones(args.hc_mult, args.dim)) + self.k_weight = nn.Parameter(torch.ones(args.hc_mult, args.dim)) + + def forward(self, x: torch.Tensor, hash_ids: torch.Tensor, token_mask: torch.Tensor | None = None) -> torch.Tensor: + """x: [B, L, hc_mult, dim]; hash_ids: [B, L, n_hash_cols]; token_mask: [B, L], False shuts + the gate so those positions pass through untouched.""" + kv = self.wkv(self.embed(hash_ids).flatten(-2)) + key, value = kv.split([self.hc_mult * self.dim, self.dim], dim=-1) + key = key.float().unflatten(-1, (self.hc_mult, self.dim)) + weight = self.q_weight.float() * self.k_weight.float() # only ever used as a product + h, eps = x.float(), self.eps + # normalized per (token, hc copy) over `dim`, NOT jointly over the copies + rstd = torch.rsqrt(h.square().mean(-1) + eps) * torch.rsqrt(key.square().mean(-1) + eps) + dot = (h * weight * key).sum(-1) * rstd * self.dim**-0.5 + # signed sqrt before the sigmoid, matching the training kernel + gate = torch.sigmoid(torch.copysign(dot.abs().clamp_min(self.clamp_value).sqrt(), dot)) + if token_mask is not None: + gate = gate.masked_fill(~token_mask.unsqueeze(-1), 0) + return (h + gate.unsqueeze(-1) * value.float().unsqueeze(-2)).to(x.dtype) + + +@lru_cache(2) +def precompute_freqs_cis(dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow) -> torch.Tensor: + """Rotary frequencies as complex exponentials, one row per position. + + With original_seq_len > 0 this applies YaRN: dimensions whose wavelength already fits inside the + training context keep their frequency, those far beyond it are divided by `factor`, and the + `beta_fast`..`beta_slow` band in between is faded across with a linear ramp. + """ + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + if original_seq_len > 0: + # the dim whose wavelength completes `rotations` turns over the training context + def corrected_dim(rotations): + return dim * math.log(original_seq_len / (rotations * 2 * math.pi)) / (2 * math.log(base)) + + low = max(math.floor(corrected_dim(beta_fast)), 0) + high = min(math.ceil(corrected_dim(beta_slow)), dim - 1) + ramp = ((torch.arange(dim // 2, dtype=torch.float32) - low) / max(high - low, 1e-3)).clamp(0, 1) + smooth = 1 - ramp + freqs = freqs / factor * (1 - smooth) + freqs * smooth + + freqs = torch.outer(torch.arange(seqlen), freqs) + return torch.polar(torch.ones_like(freqs), freqs) + + +def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor, inverse: bool = False) -> torch.Tensor: + """Rotate `x` in place, taking adjacent element pairs as complex numbers. Accepts [b, s, d] and + [b, s, h, d]; `inverse` conjugates the rotation, which is how the attention output gets the + query's rotation removed again so the cache can stay in one shared rotated form.""" + y = x + x = torch.view_as_complex(x.float().unflatten(-1, (-1, 2))) + if inverse: + freqs_cis = freqs_cis.conj() + if x.ndim == 3: + freqs_cis = freqs_cis.view(1, x.size(1), x.size(-1)) + else: + freqs_cis = freqs_cis.view(1, x.size(1), 1, x.size(-1)) + x = torch.view_as_real(x * freqs_cis).flatten(-2) + y.copy_(x) + return y + + +@lru_cache(1) +def get_window_topk_idxs(window_size: int, bsz: int, seqlen: int, start_pos: int): + """Which sliding-window cache slots each query attends to; -1 marks a slot holding nothing. + + The cache is a ring of `window_size` slots. Prefill needs one row per query, each seeing its own + causal window. A decode step has a single query that sees the whole ring, listed oldest first. + Order within a row does not matter to `sparse_attn`, which handles every slot independently. + """ + if start_pos == 0: + end = torch.arange(seqlen).unsqueeze(1) + idxs = (end - window_size + 1).clamp(0) + torch.arange(min(seqlen, window_size)) + idxs = torch.where(idxs > end, -1, idxs) # before the sequence started + else: + oldest = start_pos % window_size + 1 + idxs = torch.cat([torch.arange(oldest, window_size), torch.arange(oldest)]) + idxs = torch.where(idxs > start_pos, -1, idxs) # ring still filling + # sparse_attn needs real [b, m, topk] int32 memory, hence the materializing expand + return idxs.int().unsqueeze(0).expand(bsz, -1, -1).contiguous() + + +class Compressor(nn.Module): + """Pools `compress_ratio` consecutive tokens into one KV latent with a learned softmax gate. + + Returns the latent before RoPE, or None while a group is still filling up -- so during decode it + only yields every `compress_ratio` steps, holding the partial group in `kv_state`/`score_state`. + Pre-RoPE is deliberate: the indexer needs the unrotated form, so Attention rotates afterwards. + """ + + def __init__(self, args: ModelArgs, layer_id: int): + super().__init__() + compress_ratio = args.compress_ratios[layer_id] + head_dim = args.head_dim + self.compress_ratio = compress_ratio + self.head_dim = head_dim + self.norm = RMSNorm(head_dim, args.norm_eps) + # ratio 1 is a plain projection, so it stays in the checkpoint's bf16; the softmax pooling + # above ratio 1 runs in fp32, so those weights are promoted to fp32 to match + self.wkv = Linear(args.dim, head_dim, dtype=torch.float32 if compress_ratio > 1 else torch.bfloat16) + if compress_ratio > 1: + self.wgate = Linear(args.dim, head_dim, dtype=torch.float32) + # tail of an incomplete group, carried across decode steps + self.kv_state: torch.Tensor + self.score_state: torch.Tensor + state_shape = (args.max_batch_size, compress_ratio, head_dim) + self.register_buffer("kv_state", torch.zeros(state_shape, dtype=torch.float32), persistent=False) + self.register_buffer( + "score_state", torch.full(state_shape, -torch.inf, dtype=torch.float32), persistent=False + ) + + def forward(self, x: torch.Tensor, start_pos: int) -> torch.Tensor | None: + bsz, seqlen, _ = x.size() + ratio, dtype = self.compress_ratio, x.dtype + if ratio == 1: # one token per group: nothing to pool, so no gate and no fp32 + return self.norm(self.wkv(x)) + + x = x.float() + kv, score = self.wkv(x), self.wgate(x) + if start_pos == 0: + should_compress = seqlen >= ratio + remainder = seqlen % ratio + cutoff = seqlen - remainder + if remainder: # trailing partial group waits in the state + kv, self.kv_state[:bsz, :remainder] = kv.split([cutoff, remainder], dim=1) + score, self.score_state[:bsz, :remainder] = score.split([cutoff, remainder], dim=1) + kv = kv.unflatten(1, (-1, ratio)) + score = score.unflatten(1, (-1, ratio)) + kv = (kv * score.softmax(dim=2)).sum(dim=2) + else: # one token per step: fill a slot, and pool only when the group just completed + should_compress = (start_pos + 1) % ratio == 0 + slot = start_pos % ratio + self.kv_state[:bsz, slot] = kv.squeeze(1) + self.score_state[:bsz, slot] = score.squeeze(1) + if should_compress: + kv = (self.kv_state[:bsz] * self.score_state[:bsz].softmax(dim=1)).sum(dim=1, keepdim=True) + if not should_compress: + return None + return self.norm(kv.to(dtype)) + + +class Indexer(torch.nn.Module): + """Keeps the `index_topk` best compressed positions per query. + + A small side attention: fp4 query heads against one shared key per compressed position, scores + rectified then combined by `weights_proj`. With a candidate source this is the second of two + levels; `select_candidate_blocks` is the first. + """ + + def __init__(self, args: ModelArgs, layer_id: int): + super().__init__() + # the index keys are derived from the compressor's latent, so only a layer that compresses + # its own KV can produce them; every other indexer reads them from that layer's cache + self.owns_k = layer_id in args.kv_source_layers + self.compress_ratio = args.compress_ratios[layer_id] + self.is_candidate_source = layer_id == args.candidate_source_layer + self.uses_candidates = 0 <= args.candidate_source_layer < layer_id + self.candidate_topk_blocks = args.candidate_topk_blocks + self.candidate_block_size = args.candidate_block_size + self.dim = args.dim + self.n_heads = args.index_n_heads + self.n_local_heads = args.index_n_heads // world_size + self.index_head_dim = args.index_head_dim + self.rope_head_dim = args.rope_head_dim + self.index_topk = args.index_topk + self.q_lora_rank = args.q_lora_rank + self.softmax_scale = self.index_head_dim**-0.5 + self.wq_b = ColumnParallelLinear(self.q_lora_rank, self.n_heads * self.index_head_dim) + self.weights_proj = ColumnParallelLinear(self.dim, self.n_heads, dtype=torch.bfloat16) + self.freqs_cis: torch.Tensor | None = None + if self.owns_k: + self.wk = Linear(args.head_dim, self.index_head_dim, dtype=torch.bfloat16) + self.k_norm = RMSNorm(self.index_head_dim, args.norm_eps) + self.k_cache: torch.Tensor + self.register_buffer( + "k_cache", + torch.zeros(args.max_batch_size, args.max_seq_len // self.compress_ratio, args.index_head_dim), + persistent=False, + ) + + def forward(self, x: torch.Tensor, qr: torch.Tensor, latent: torch.Tensor, start_pos: int, offset: int): + """`latent` is this layer's RoPE-free compressed latent, None when this layer does not + compress or when its current group is still incomplete. An index-key owner turns it into + index keys here, which has to happen before Attention overwrites that same storage with + the RoPE'd, quantized values.""" + assert self.freqs_cis is not None + bsz, seqlen, _ = x.size() + ratio, rd, end_pos = self.compress_ratio, self.rope_head_dim, start_pos + seqlen + + # latent is None while a group is still filling up, so there is nothing to publish yet + if self.owns_k and latent is not None: + # a latent stands for the first token of its group, so group j takes position j * ratio + freqs = ( + self.freqs_cis[: seqlen - seqlen % ratio : ratio] + if start_pos == 0 + else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0) + ) + k = self.k_norm(self.wk(latent)) + apply_rotary_emb(k[..., -rd:], freqs) + fp4_act_quant(k, fp4_block_size, True) + self.k_cache[:bsz, start_pos // ratio : start_pos // ratio + k.size(1)] = k + shared_attn.index_k = self.k_cache + + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.index_head_dim)) + apply_rotary_emb(q[..., -rd:], self.freqs_cis[start_pos:end_pos]) + fp4_act_quant(q, fp4_block_size, True) + + index_k = shared_attn.index_k[:bsz, : end_pos // ratio] + weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads**-0.5) + index_score = torch.einsum("bshd,btd->bsht", q, index_k) + index_score = (index_score.relu_() * weights.unsqueeze(-1)).sum(dim=2) + if world_size > 1: + dist.all_reduce(index_score) + + # how many compressed positions each query can see: a block becomes visible once the query + # has passed its last token. One query per decode step, so there it is just a number. + if start_pos == 0: + compress_lens = (torch.arange(1, seqlen + 1, device=x.device) // ratio).unsqueeze(-1) + index_score.masked_fill_(torch.arange(seqlen // ratio, device=x.device) >= compress_lens, -torch.inf) + else: + compress_lens = end_pos // ratio + + if self.is_candidate_source: + shared_attn.candidates = select_candidate_blocks( + index_score, compress_lens, self.candidate_topk_blocks, self.candidate_block_size + ) + elif self.uses_candidates: + # level two: score with our own weights, but only inside the source's candidate blocks + index_score = index_score.masked_fill(~shared_attn.candidates, -torch.inf) + + # top-k by score, re-sorted into position order; unreachable -> -1, rest shifted by offset + topk = min(self.index_topk, end_pos // ratio) + idxs = index_score.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values + return torch.where(idxs < compress_lens, idxs + offset, -1).int() + + +def select_candidate_blocks( + logits: torch.Tensor, + compress_lens: torch.Tensor | int, + topk_blocks: int, + block_size: int, +) -> torch.Tensor: + """Level one of the two-level top-k: keep the `topk_blocks` highest-scoring blocks per query. + + `logits` is [..., n_positions] with positions the query cannot reach already at -inf, which is + what makes a block score of -inf mean "not reachable yet". `compress_lens` is a plain int during + decode, or broadcasts against logits' leading dims during prefill. Returns a bool mask shaped + like `logits`, so the layers consuming it just mask and never think about blocks again. + """ + width = logits.size(-1) + # score each block by its best position; -inf pads the last one out to block_size + scores = F.pad(logits, (0, -width % block_size), value=-torch.inf) + scores = scores.unflatten(-1, (-1, block_size)).amax(dim=-1) + num_blocks = scores.size(-1) + + # the block with this query's newest position is only partly filled, so pin it in: it holds the + # most recent tokens but could otherwise be outscored by an older, full block + last = (compress_lens - 1) // block_size + scores = scores.masked_fill(torch.arange(num_blocks, device=logits.device) == last, torch.inf) + + top = scores.topk(min(topk_blocks, num_blocks), dim=-1) + # fewer reachable blocks than topk_blocks means leftover picks came back -inf: drop them + keep = torch.zeros_like(scores, dtype=torch.bool).scatter_(-1, top.indices, top.values > -torch.inf) + return keep.repeat_interleave(block_size, dim=-1)[..., :width] + + +class Attention(nn.Module): + """Latent attention over two KV sources at once, concatenated into one `sparse_attn` call: a + sliding window of raw KV, plus -- when compress_ratio > 0 -- `index_topk` compressed positions + reaching further back. Q and the output projection are both low-rank, the latter grouped. + + compress_ratio > 0 does not mean the layer compresses its own KV: only kv_source_layers do, + the rest read that same cache. + """ + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__() + self.layer_id = layer_id + self.dim = args.dim + self.n_heads = args.n_heads + self.n_local_heads = args.n_heads // world_size + self.q_lora_rank = args.q_lora_rank + self.o_lora_rank = args.o_lora_rank + self.head_dim = args.head_dim + self.rope_head_dim = args.rope_head_dim + self.nope_head_dim = args.head_dim - args.rope_head_dim + self.n_groups = args.o_groups + self.n_local_groups = self.n_groups // world_size + self.window_size = args.window_size + self.compress_ratio = args.compress_ratios[layer_id] + self.eps = args.norm_eps + + self.attn_sink = nn.Parameter(torch.empty(self.n_local_heads, dtype=torch.float32)) + self.wq_a = Linear(self.dim, self.q_lora_rank) + self.q_norm = RMSNorm(self.q_lora_rank, self.eps) + self.wq_b = ColumnParallelLinear(self.q_lora_rank, self.n_heads * self.head_dim) + self.wkv = Linear(self.dim, self.head_dim) + self.kv_norm = RMSNorm(self.head_dim, self.eps) + self.wo_a = ColumnParallelLinear( + self.n_heads * self.head_dim // self.n_groups, + self.n_groups * args.o_lora_rank, + dtype=torch.bfloat16, + ) + self.wo_b = RowParallelLinear(self.n_groups * args.o_lora_rank, self.dim) + self.softmax_scale = self.head_dim**-0.5 + + is_backbone = layer_id < args.n_layers + self.is_kv_source = is_backbone and layer_id in args.kv_source_layers + self.is_index_source = is_backbone and layer_id in args.index_source_layers + self.compressor: Compressor | None = None + self.indexer: Indexer | None = None + if self.is_kv_source: + self.compressor = Compressor(args, layer_id) + if self.is_index_source: + self.indexer = Indexer(args, layer_id) + + self.window_kv_cache: torch.Tensor + self.register_buffer( + "window_kv_cache", + torch.zeros(args.max_batch_size, args.window_size, self.head_dim), + persistent=False, + ) + if self.is_kv_source: + self.compress_kv_cache: torch.Tensor + self.register_buffer( + "compress_kv_cache", + torch.zeros( + args.max_batch_size, + args.max_seq_len // self.compress_ratio, + self.head_dim, + ), + persistent=False, + ) + if self.compress_ratio: + original_seq_len, rope_theta = ( + args.original_seq_len, + args.compress_rope_theta, + ) + else: + # disable YaRN and use base rope_theta in pure sliding-window attention + original_seq_len, rope_theta = 0, args.rope_theta + freqs_cis = precompute_freqs_cis( + self.rope_head_dim, + args.max_seq_len, + original_seq_len, + rope_theta, + args.rope_factor, + args.beta_fast, + args.beta_slow, + ) + self.freqs_cis: torch.Tensor + self.register_buffer("freqs_cis", freqs_cis, persistent=False) + + def _window_kv(self, x, freqs_cis, start_pos): + """This layer's sliding-window K and the window positions every query may attend to. The K + stays fp8, quantized over the whole post-RoPE vector, RoPE tail included.""" + bsz, seqlen, _ = x.size() + win = self.window_size + kv = self.kv_norm(self.wkv(x)) + apply_rotary_emb(kv[..., -self.rope_head_dim :], freqs_cis) + act_quant(kv, fp8_block_size, scale_fmt, scale_dtype, True) + if start_pos == 0: # prefill: attend over this chunk, seeding the ring buffer for decode + if seqlen <= win: + self.window_kv_cache[:bsz, :seqlen] = kv + else: + cutoff = seqlen % win + self.window_kv_cache[:bsz, cutoff:win], self.window_kv_cache[:bsz, :cutoff] = kv[:, -win:].split( + [win - cutoff, cutoff], dim=1 + ) + window_kv = kv + else: # decode: one token into the ring buffer, attend over the whole window + self.window_kv_cache[:bsz, start_pos % win] = kv.squeeze(1) + window_kv = self.window_kv_cache[:bsz] + return window_kv, get_window_topk_idxs(win, bsz, seqlen, start_pos) + + def _compress_topk_idxs(self, x, qr, latent, start_pos, offset, compress_len): + """Which compressed positions each query attends to. Index sources run their own indexer; + the layers in between reuse the result their source published.""" + if not self.is_index_source: + return shared_attn.topk_idxs + + bsz, seqlen, _ = x.size() + if compress_len == 0: + idxs = torch.empty(bsz, seqlen, 0, dtype=torch.int32, device=x.device) + else: + assert self.indexer is not None + if self.indexer.freqs_cis is None: + self.indexer.freqs_cis = self.freqs_cis + idxs = self.indexer(x, qr, latent, start_pos, offset) + shared_attn.topk_idxs = idxs + return idxs + + def _compress_kv(self, x, qr, start_pos, offset): + """The shared compressed KV and the compressed positions every query may attend to. This + layer compresses its own KV only when it is a source; otherwise it just reads the cache.""" + bsz, seqlen, _ = x.size() + ratio = self.compress_ratio + compress_len = (start_pos + seqlen) // ratio + latent = None + if self.is_kv_source: + latent = self.compressor(x, start_pos) + shared_attn.compress_kv = self.compress_kv_cache + # the indexer needs the latent before RoPE, so it runs before the cache is written + idxs = self._compress_topk_idxs(x, qr, latent, start_pos, offset, compress_len) + if latent is not None: + # a latent stands for the first token of its group, so group j takes position j * ratio + freqs = ( + self.freqs_cis[: seqlen - seqlen % ratio : ratio] + if start_pos == 0 + else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0) + ) + apply_rotary_emb(latent[..., -self.rope_head_dim :], freqs) + # Compressed KV uses groups of 16 with E4M3 scales; the indexer uses 32 with E8M0. + fp4_act_quant(latent, 16, True, scale_dtype=torch.float8_e4m3fn) + self.compress_kv_cache[:bsz, start_pos // ratio : start_pos // ratio + latent.size(1)] = latent + # read after the write, so this does not depend on the slice aliasing the cache + return shared_attn.compress_kv[:bsz, :compress_len], idxs + + def forward(self, x: torch.Tensor, start_pos: int): + bsz, seqlen, _ = x.size() + freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen] + rd = self.rope_head_dim + + qr = self.q_norm(self.wq_a(x)) + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.head_dim)) + apply_rotary_emb(q[..., -rd:], freqs_cis) + + kv, topk_idxs = self._window_kv(x, freqs_cis, start_pos) + if self.compress_ratio: + compress_kv, compress_idxs = self._compress_kv(x, qr, start_pos, kv.size(1)) + kv = torch.cat([kv, compress_kv], dim=1) + topk_idxs = torch.cat([topk_idxs, compress_idxs], dim=-1) + + o = sparse_attn(q, kv, self.attn_sink, topk_idxs, self.softmax_scale) + apply_rotary_emb(o[..., -rd:], freqs_cis, True) + + # wo_a is block-diagonal over groups (each projects only its own heads), hence einsum not + # Linear. convert.py dequantizes it to bf16; an fp8 grouped GEMM would halve the memory. + o = o.view(bsz, seqlen, self.n_local_groups, -1) + wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) + o = torch.einsum("bsgd,grd->bsgr", o, wo_a) + x = self.wo_b(o.flatten(2)) + return x + + +class Gate(nn.Module): + """MoE gating. The correction bias steers expert selection only; the routing weights come from the + unbiased scores. Image-span tokens use a separate bias (training `noaux_tc_for_vl`).""" + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__() + n_routed_experts, n_activated_experts = args.get_moe_config(layer_id) + self.dim = args.dim + self.topk = n_activated_experts + self.score_func = args.score_func + self.gate_temp = args.gate_temp + self.norm_topk_prob = args.norm_topk_prob + self.route_scale = args.route_scale + self.weight = nn.Parameter(torch.empty(n_routed_experts, args.dim)) + self.bias = nn.Parameter(torch.empty(n_routed_experts, dtype=torch.float32)) + self.bias_vl = nn.Parameter(torch.empty(n_routed_experts, dtype=torch.float32)) if args.vision_enabled else None + + def forward(self, x: torch.Tensor, image_mask: torch.Tensor | None = None) -> tuple[torch.Tensor, torch.Tensor]: + """x: [n, dim]; image_mask: [n] bool, True for tokens inside an image span.""" + scores = linear(x.float(), self.weight.float()) / self.gate_temp + if self.score_func == "softmax": + scores = scores.softmax(dim=-1) + elif self.score_func == "sigmoid": + scores = scores.sigmoid() + else: + scores = F.softplus(scores).sqrt() + bias = self.bias + if image_mask is not None and self.bias_vl is not None: + bias = torch.where(image_mask.unsqueeze(-1), self.bias_vl, bias) + # the bias picks experts but does not scale them: weights come from the raw scores + indices = (scores + bias).topk(self.topk, dim=-1)[1] + weights = scores.gather(1, indices) + if self.norm_topk_prob and self.topk > 1: + weights /= weights.sum(dim=-1, keepdim=True) + 1e-20 # not norm_eps, matches training + weights *= self.route_scale + return weights, indices + + +class Expert(nn.Module): + """One SwiGLU FFN. The clamps come straight from training, where they keep fp8/fp4 activations in + range: the up branch is clamped on both sides, the gate branch only from above.""" + + def __init__(self, dim: int, inter_dim: int, dtype=None, swiglu_limit=0.0): + super().__init__() + self.w1 = Linear(dim, inter_dim, dtype=dtype) + self.w2 = Linear(inter_dim, dim, dtype=dtype) + self.w3 = Linear(dim, inter_dim, dtype=dtype) + self.swiglu_limit = swiglu_limit + + def forward(self, x: torch.Tensor, weights: torch.Tensor | None = None) -> torch.Tensor: + dtype = x.dtype + gate = self.w1(x).float() + up = self.w3(x).float() + if self.swiglu_limit > 0: + up = torch.clamp(up, min=-self.swiglu_limit, max=self.swiglu_limit) + gate = torch.clamp(gate, max=self.swiglu_limit) + x = F.silu(gate) * up + if weights is not None: + x = weights * x + return self.w2(x.to(dtype)) + + +class MoE(nn.Module): + """Top-k routed experts plus one shared expert every token goes through. Experts are split + across ranks, so `self.experts` is None for those another rank owns.""" + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__() + n_routed_experts, n_activated_experts = args.get_moe_config(layer_id) + self.layer_id = layer_id + self.dim = args.dim + assert n_routed_experts % world_size == 0, ( + f"Number of experts must be divisible by world size (world_size={world_size})" + ) + self.n_routed_experts = n_routed_experts + self.n_local_experts = n_routed_experts // world_size + self.n_activated_experts = n_activated_experts + self.experts_start_idx = rank * self.n_local_experts + self.experts_end_idx = self.experts_start_idx + self.n_local_experts + self.gate = Gate(layer_id, args) + expert_dtype = torch.float4_e2m1fn_x2 if args.expert_dtype == "fp4" else None + self.experts = nn.ModuleList( + [ + Expert( + args.dim, + args.moe_inter_dim, + dtype=expert_dtype, + swiglu_limit=args.swiglu_limit, + ) + if self.experts_start_idx <= i < self.experts_end_idx + else None + for i in range(self.n_routed_experts) + ] + ) + assert args.n_shared_experts == 1 + self.shared_experts = Expert(args.dim, args.moe_inter_dim, swiglu_limit=args.swiglu_limit) + + def forward(self, x: torch.Tensor, image_mask: torch.Tensor | None = None) -> torch.Tensor: + shape = x.size() + x = x.view(-1, self.dim) + weights, indices = self.gate(x, None if image_mask is None else image_mask.flatten()) + y = torch.zeros_like(x, dtype=torch.float32) + counts = torch.bincount(indices.flatten(), minlength=self.n_routed_experts).tolist() + for i in range(self.experts_start_idx, self.experts_end_idx): + if counts[i] == 0: + continue + expert = self.experts[i] + idx, top = torch.where(indices == i) + y[idx] += expert(x[idx], weights[idx, top, None]) + if world_size > 1: + dist.all_reduce(y) + y += self.shared_experts(x) + return y.type_as(x).view(shape) + + +class Block(nn.Module): + """A block whose residual stream is `hc_mult` parallel copies (Hyper-Connections). + + Attention and FFN each sit between `hc_pre` (collapse the copies into one sublayer input) and + `hc_post` (expand back out, mixing the residual in through `comb`). `hc_mixes` derives all three + coefficient sets from the stream itself, `comb` made doubly stochastic by Sinkhorn. + + The coefficients a sublayer computes are used by the *next* one -- see `forward`. + """ + + attention_cls = Attention + + def __init__( + self, + layer_id: int, + args: ModelArgs, + engram_layout: EngramLayout | None = None, + ): + super().__init__() + self.layer_id = layer_id + self.norm_eps = args.norm_eps + self.attn = self.attention_cls(layer_id, args) + self.ffn = MoE(layer_id, args) + self.engram = None + if engram_layout is not None and layer_id in engram_layout.layer_ids: + self.engram = Engram(args, layer_id, engram_layout) + self.attn_norm = RMSNorm(args.dim, self.norm_eps) + self.ffn_norm = RMSNorm(args.dim, self.norm_eps) + self.hc_mult = hc_mult = args.hc_mult + self.hc_sinkhorn_iters = args.hc_sinkhorn_iters + self.hc_eps = args.hc_eps + mix_hc = (2 + hc_mult) * hc_mult + hc_dim = hc_mult * args.dim + with set_dtype(torch.float32): + self.hc_attn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim)) + self.hc_ffn_fn = nn.Parameter(torch.empty(mix_hc, hc_dim)) + self.hc_attn_base = nn.Parameter(torch.empty(mix_hc)) + self.hc_ffn_base = nn.Parameter(torch.empty(mix_hc)) + self.hc_attn_scale = nn.Parameter(torch.empty(3)) + self.hc_ffn_scale = nn.Parameter(torch.empty(3)) + + def hc_mixes(self, x: torch.Tensor, hc_fn: torch.Tensor, hc_scale: torch.Tensor, hc_base: torch.Tensor): + """x: [b,s,hc,d], hc_fn: [mix_hc, hc*d], hc_scale: [3], hc_base: [mix_hc]. Returns the + pre / post / comb coefficients, split out of one projection of the flattened stream.""" + # normalized over the whole flattened hc*d stream, one statistic per token + x = x.flatten(2).float() + rsqrt = torch.rsqrt(x.square().mean(-1, keepdim=True) + self.norm_eps) + mixes = F.linear(x, hc_fn) * rsqrt + return hc_split_sinkhorn(mixes, hc_scale, hc_base, self.hc_mult, self.hc_sinkhorn_iters, self.hc_eps) + + def hc_pre(self, x: torch.Tensor, pre_mix: torch.Tensor): + """Collapse the hc copies into one, weighted by pre_mix. [b,s,hc,d] x [b,s,hc] -> [b,s,d]""" + y = torch.sum(pre_mix.unsqueeze(-1) * x.float(), dim=2) + return y.to(x.dtype) + + def hc_post(self, x: torch.Tensor, residual: torch.Tensor, post: torch.Tensor, comb: torch.Tensor): + """Expand the sublayer output back to hc copies and mix the residual in through `comb`. + x: [b,s,d], residual: [b,s,hc,d], post: [b,s,hc], comb: [b,s,hc,hc] -> [b,s,hc,d]""" + y = post.unsqueeze(-1) * x.unsqueeze(-2) + torch.sum(comb.unsqueeze(-1) * residual.unsqueeze(-2), dim=2) + return y.type_as(x) + + def forward( + self, + x: torch.Tensor, + start_pos: int, + pre_mix: torch.Tensor, + image_mask: torch.Tensor | None, + *attn_args, + ) -> tuple[torch.Tensor, torch.Tensor]: + """`pre_mix` collapses the hc_mult copies down to one input for this block's attention. Each + sub-block's own `hc_mixes` produces the mix for the *next* one, so attention uses what the + previous layer's FFN produced and the FFN uses what this attention produced. + + image_mask: [b, s] bool, True inside image spans (selects the VL routing bias).""" + residual = x + attn_pre, attn_post, attn_comb = self.hc_mixes(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base) + x = self.hc_pre(x, pre_mix) + x = self.attn_norm(x) + x = self.attn(x, start_pos, *attn_args) + x = self.hc_post(x, residual, attn_post, attn_comb) + + residual = x + ffn_pre, ffn_post, ffn_comb = self.hc_mixes(x, self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base) + x = self.hc_pre(x, attn_pre) + x = self.ffn_norm(x) + x = self.ffn(x, image_mask) + x = self.hc_post(x, residual, ffn_post, ffn_comb) + return x, ffn_pre + + +class ParallelHead(nn.Module): + def __init__(self, vocab_size: int, dim: int, norm_eps: float = 1e-6, hc_eps: float = 1e-6): + super().__init__() + self.vocab_size = vocab_size + self.dim = dim + self.norm_eps = norm_eps + self.hc_eps = hc_eps + self.part_vocab_size = vocab_size // world_size + # bf16 in the checkpoint, kept as fp32 here so the logits come out in fp32 directly + self.weight = nn.Parameter(torch.empty(self.part_vocab_size, self.dim, dtype=torch.float32)) + + def forward(self, x: torch.Tensor, full_logits=False): + """x: [b, s, d]. Generation only needs the last position, so that is the default.""" + if not full_logits: + x = x[:, -1] + logits = F.linear(x.float(), self.weight) + if world_size > 1: + all_logits = [torch.empty_like(logits) for _ in range(world_size)] + dist.all_gather(all_logits, logits) + logits = torch.cat(all_logits, dim=-1) + return logits + + +@lru_cache(1) +def get_dspark_topk_idxs(window_size: int, bsz: int, block_size: int, start_pos: int): + assert start_pos > 0 + matrix = torch.cat( + [ + torch.arange(min(window_size, start_pos + 1)), + window_size + torch.arange(block_size), + ] + ) + return matrix.int().view(1, 1, -1).expand(bsz, block_size, -1).contiguous() + + +class DSparkAttention(Attention): + def forward(self, x: torch.Tensor, start_pos: int, main_x: torch.Tensor): + assert self.compress_ratio == 0 + bsz, seqlen, _ = main_x.size() + win = self.window_size + rd = self.rope_head_dim + + main_freqs_cis = self.freqs_cis[start_pos : start_pos + seqlen] + main_kv = self.kv_norm(self.wkv(main_x)) + apply_rotary_emb(main_kv[..., -rd:], main_freqs_cis) + act_quant(main_kv, fp8_block_size, scale_fmt, scale_dtype, True) + + if start_pos == 0: + if seqlen <= win: + self.window_kv_cache[:bsz, :seqlen] = main_kv + else: + cutoff = seqlen % win + self.window_kv_cache[:bsz, cutoff:win], self.window_kv_cache[:bsz, :cutoff] = main_kv[:, -win:].split( + [win - cutoff, cutoff], dim=1 + ) + return x + + bsz, block_size, _ = x.size() + freqs_cis = self.freqs_cis[start_pos + seqlen : start_pos + seqlen + block_size] + + qr = self.q_norm(self.wq_a(x)) + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.head_dim)) + apply_rotary_emb(q[..., -rd:], freqs_cis) + kv = self.kv_norm(self.wkv(x)) + apply_rotary_emb(kv[..., -rd:], freqs_cis) + act_quant(kv, fp8_block_size, scale_fmt, scale_dtype, True) + + topk_idxs = get_dspark_topk_idxs(win, bsz, block_size, start_pos) + self.window_kv_cache[:bsz, start_pos % win] = main_kv.squeeze(1) + kv = torch.cat([self.window_kv_cache[:bsz], kv], dim=1) + o = sparse_attn(q, kv, self.attn_sink, topk_idxs, self.softmax_scale) + apply_rotary_emb(o[..., -rd:], freqs_cis, True) + + o = o.view(bsz, block_size, self.n_local_groups, -1) + wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) + o = torch.einsum("bsgd,grd->bsgr", o, wo_a) + x = self.wo_b(o.flatten(2)) + return x + + +class DSparkMarkovHead(nn.Module): + def __init__(self, vocab_size: int, dspark_markov_rank: int): + super().__init__() + self.embed = ParallelEmbedding(vocab_size, dspark_markov_rank) + self.head = ParallelHead(vocab_size, dspark_markov_rank) + + def forward(self, token_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + embed = self.embed(token_ids) + logits = self.head(embed, full_logits=True) + return logits, embed + + +class DSparkConfidenceHead(nn.Module): + def __init__(self, input_dim: int): + super().__init__() + # proj in the checkpoint is stored in bf16, while the parameter here is stored in fp32 for fp32 confidence score. + self.proj = Linear(input_dim, 1, dtype=torch.float32) + + def forward(self, hidden: torch.Tensor, markov_embed: torch.Tensor): + hidden = torch.cat([hidden, markov_embed], dim=-1) + return self.proj(hidden.float()).squeeze(-1) + + +class DSparkBlock(Block): + """DSpark stage stored under the mtp.* checkpoint namespace.""" + + attention_cls = DSparkAttention + + def __init__(self, layer_id: int, args: ModelArgs): + super().__init__(layer_id, args) + stage_id = layer_id - args.n_layers + self.block_size = args.dspark_block_size + self.noise_token_id = args.dspark_noise_token_id + self.temperature = args.temperature + if stage_id == 0: + assert len(args.dspark_target_layer_ids) > 0, "DSpark needs target layers" + self.main_proj = Linear(args.dim * len(args.dspark_target_layer_ids), args.dim) + self.main_norm = RMSNorm(args.dim, args.norm_eps) + if stage_id == args.n_mtp_layers - 1: + self.norm = RMSNorm(args.dim, args.norm_eps) + self.markov_head = DSparkMarkovHead(args.vocab_size, args.dspark_markov_rank) + self.confidence_head = DSparkConfidenceHead(args.dim + args.dspark_markov_rank) + self.embed: ParallelEmbedding | None = None + self.head: ParallelHead | None = None + + def forward(self, x: torch.Tensor, start_pos: int, pre_mix: torch.Tensor, main_x: torch.Tensor): + if start_pos == 0: + self.attn(x, start_pos, main_x) # prefill only seeds the window KV cache + return x, pre_mix + return super().forward(x, start_pos, pre_mix, None, main_x) # drafts are text: no VL bias + + def forward_embed(self, main_hidden: torch.Tensor, input_ids: torch.Tensor): + assert self.embed is not None + main_x = self.main_norm(self.main_proj(main_hidden)) + draft_input_ids = input_ids.new_full([input_ids.size(0), self.block_size], self.noise_token_id) + draft_input_ids[:, 0] = input_ids + x = self.embed(draft_input_ids) + x = x.unsqueeze(2).repeat(1, 1, self.hc_mult, 1) + return x, main_x + + def forward_head( + self, + x: torch.Tensor, + pre_mix: torch.Tensor, + input_ids: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert self.head is not None + x = self.hc_pre(x, pre_mix) + logits = self.head(self.norm(x), full_logits=True) + output_ids = input_ids.new_empty(input_ids.size(0), self.block_size + 1) + output_ids[:, 0] = input_ids + markov_embeds = [] + for i in range(self.block_size): + logits_bias, markov_embed = self.markov_head(output_ids[:, i]) + logits[:, i].add_(logits_bias) + markov_embeds.append(markov_embed) + output_ids[:, i + 1] = sample(logits[:, i], self.temperature) + markov_embed = torch.stack(markov_embeds, dim=1) + confidence = self.confidence_head(x, markov_embed) + return output_ids, logits, confidence + + +def make_identity_pre_mix(x: torch.Tensor, hc_mult: int) -> torch.Tensor: + """initial one-hot mix""" + pre_mix = x.new_zeros(x.size(0), x.size(1), hc_mult, dtype=torch.float32) + pre_mix[:, :, 0] = 1.0 + return pre_mix + + +class SharedAttentionRuntime: + """What attention layers hand down the stack instead of recomputing. Layers run in order and + every source writes before its consumers read, so one slot each is enough and nothing needs + resetting between forwards. Sources: compress_kv and index_k from kv_source_layers, topk_idxs + from index_source_layers, candidates from candidate_source_layer.""" + + def __init__(self): + self.compress_kv: torch.Tensor | None = None + self.index_k: torch.Tensor | None = None + self.topk_idxs: torch.Tensor | None = None + self.candidates: torch.Tensor | None = None + + +# Only ever one model per process, same as world_size / rank / default_dtype above. +shared_attn = SharedAttentionRuntime() + + +class Transformer(nn.Module): + """DeepSeek-V4.1: embed -> expand to hc_mult copies -> blocks -> collapse -> logits. Building + this sets the globals at the top of the file. The tokenizer only feeds the engram token map.""" + + def __init__(self, args: ModelArgs, tokenizer=None): + global world_size, rank, default_dtype + world_size = dist.get_world_size() if dist.is_initialized() else 1 + rank = dist.get_rank() if dist.is_initialized() else 0 + default_dtype = torch.float8_e4m3fn if args.dtype == "fp8" else torch.bfloat16 + super().__init__() + self.max_seq_len = args.max_seq_len + self.temperature = args.temperature + self.norm_eps = args.norm_eps + self.hc_eps = args.hc_eps + self.engram_layout = EngramLayout.from_args(args) + self.engram_hash = ( + NgramHashState(args, self.engram_layout, tokenizer) if self.engram_layout is not None else None + ) + self.embed = ParallelEmbedding(args.vocab_size, args.dim) + self.layers = torch.nn.ModuleList() + for layer_id in range(args.n_layers): + self.layers.append(Block(layer_id, args, self.engram_layout)) + self.norm = RMSNorm(args.dim, self.norm_eps) + self.head = ParallelHead(args.vocab_size, args.dim, self.norm_eps, self.hc_eps) + self.mtp = torch.nn.ModuleList() + self.target_layer_ids = args.dspark_target_layer_ids + if args.dspark_block_size: + for layer_id in range(args.n_mtp_layers): + self.mtp.append(DSparkBlock(args.n_layers + layer_id, args)) + self.mtp[-1].embed = self.embed + self.mtp[-1].head = self.head + self.hc_mult = args.hc_mult + self.vision = None + if args.vision_enabled: + self.vision = ViT(args) + self.aligner = Aligner(args) + # learned embeddings for the image span delimiters + self.image_start = nn.Parameter(torch.empty(args.dim)) + self.image_end = nn.Parameter(torch.empty(args.dim)) + self.image_newline = nn.Parameter(torch.empty(args.dim)) + + @torch.inference_mode() + def encode_image(self, patches: torch.Tensor, n_vit_h: int, n_vit_w: int) -> torch.Tensor: + return self.aligner(self.vision(patches, n_vit_h, n_vit_w), n_vit_h, n_vit_w) + + def merge_image_embeddings(self, images, h: torch.Tensor): + """Overwrite each image's token span in h with its ViT/aligner features. The IMAGE slots take + the aligner rows in row-major order; the span delimiters take learned embeddings.""" + for i, sample in enumerate(images): + for img in sample or (): + types = img.types.to(h.device) + span = h[i, img.start : img.start + types.numel()] + span[types == IMAGE_START] = self.image_start.to(h.dtype) + span[types == IMAGE_END] = self.image_end.to(h.dtype) + span[types == IMAGE_NEW_LINE] = self.image_newline.to(h.dtype) + embeds = self.encode_image(img.patches.to(h.device), img.n_vit_h, img.n_vit_w) + span[types == IMAGE] = embeds.to(h.dtype) + + @torch.inference_mode() + def forward( + self, input_ids: torch.Tensor, start_pos: int = 0, images=None, token_types: torch.Tensor | None = None + ): + """input_ids: [b, s], every entry a real token id -- generate.py only ever passes positions it + has already filled, so the padding it uses internally never reaches here. token_types / + images carry the VL inputs built by image_processor.prepare_vl_inputs; image spans must lie + inside the first (start_pos 0) chunk.""" + image_mask = None if token_types is None else token_types >= 0 # TEXT is -1 + # image tokens take no part in an n-gram and get no engram contribution; text-only needs no mask + engram_mask = None if image_mask is None else ~image_mask + engram_hashes = self.engram_hash(input_ids, start_pos, engram_mask) if self.engram_hash is not None else None + h = self.embed(input_ids) + if images is not None: + assert start_pos == 0, "image spans must be prefilled in a single chunk" + self.merge_image_embeddings(images, h) + # Expand to hc_mult copies for Hyper-Connections + h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1) + main_hiddens = [] + pre_mix = make_identity_pre_mix(h, self.hc_mult) + for i, layer in enumerate(self.layers): + if layer.engram is not None: + h = layer.engram(h, engram_hashes[:, :, layer.engram.layer_hash_index, :], engram_mask) + # the MTP head reads the attention input of its target layers, not their output + if i in self.target_layer_ids: + main_hiddens.append(h.mean(dim=2)) + h, pre_mix = layer(h, start_pos, pre_mix, image_mask) + h = layer.hc_pre(h, pre_mix) + logits = self.head(self.norm(h)) + output_ids = sample(logits, self.temperature) + main_hidden = torch.cat(main_hiddens, dim=-1) if main_hiddens else None + return output_ids, logits, main_hidden + + @torch.inference_mode() + def forward_spec(self, input_ids: torch.Tensor, main_hidden: torch.Tensor, start_pos: int = 0): + h, main_x = self.mtp[0].forward_embed(main_hidden, input_ids) + pre_mix = make_identity_pre_mix(h, self.hc_mult) + for layer in self.mtp: + h, pre_mix = layer(h, start_pos, pre_mix, main_x) + if start_pos == 0: + return None + return self.mtp[-1].forward_head(h, pre_mix, input_ids) + + +def sample(logits, temperature: float = 1.0): + """Gumbel-max trick: equivalent to multinomial sampling but faster on GPU, + since it avoids the GPU-to-CPU sync in torch.multinomial.""" + if temperature == 0: + return logits.argmax(dim=-1) + logits = logits / max(temperature, 1e-5) + probs = torch.softmax(logits, dim=-1, dtype=torch.float32) + return probs.div_(torch.empty_like(probs).exponential_(1)).argmax(dim=-1) + + +if __name__ == "__main__": + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") + torch.manual_seed(0) + args = ModelArgs(dspark_block_size=6, dspark_target_layer_ids=(3, 4)) + x = torch.randint(0, args.vocab_size, (2, 150)) + model = Transformer(args) + + output_ids, logits, main_hidden = model(x[:, :128]) + model.forward_spec(output_ids, main_hidden) + for i in range(128, 150): + output_ids, logits, main_hidden = model(x[:, i : i + 1], i) + result = model.forward_spec(output_ids, main_hidden, i) + assert result is not None + output_ids, logits, confidence = result diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/oracle.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/oracle.py new file mode 100644 index 000000000000..386bfc0c9286 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/oracle.py @@ -0,0 +1,1241 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Trusted, cached CPU expected results for DeepSeek-V4.1 prefill (bead §6, tt-metal_tracker-8y7.7). + +The vendored reference (``model.py``) is the semantic authority; this module only builds it, observes it +with forward hooks, and caches what it saw. The reference runs single-shot (``start_pos == 0``) and is +never called with ``start_pos > 0``: chunked expectations are slices of the single-shot result +(``chunk_expectations``, graph.md §4). + +Typical use:: + + spec = real_spec((2, 3), seq_len=2048) # V4.1 layers 2 -> 3, real dims, synthetic + spec = real_spec((0, 2, 3), 2048, checkpoint=HF_SNAPSHOT) # the same with checkpoint weights + spec = real_spec((20, 36, 37, 38, 39), 512, dspark=True, checkpoint=HF_SNAPSHOT, + checkpoint_units=("mtp.0", "mtp.1", "mtp.2")) # real DSpark, synthetic backbone + spec = small_spec(seq_len=41) # small dims, all six block types + DSpark + model = build_reference(spec) # e.g. for device weights + result = oracle(spec, random_tokens(spec), model) # cached captures and state + chunks = chunk_expectations(result, chunk_len=5120, pad_multiple=2 * 32 * sp) + +Weights. Every model unit (``embed``, ``layers.``, ``norm``, ``head``, ``mtp.``) is filled by +``testing.init_weights`` from its own seed, derived from (seed, unit name, V4.1 layer id). A layer's +synthetic weights therefore do not depend on which other layers the schedule holds. Checkpoint weights +are read from the HF shards as stored (FP8/FP4 + E8M0 scales) and converted like upstream ``convert.py`` +(``wo_a`` dequantized to bf16, FP4 experts viewed as ``float4_e2m1fn_x2``). Synthetic Engram tables are +row-sparse: row ``r`` of layer ``l`` is drawn from its own seed, and only the rows a prompt hashes to are +materialized (a real table has 384M rows); lookups still run through the reference module. A spec with +``checkpoint_units`` mixes the two: the named units come from the checkpoint, every other unit is synthetic +(bit-identical to the fully synthetic spec's), e.g. real DSpark layers on a backbone whose shards are absent. + +Result layout (all tensors batch-squeezed, keyed by V4.1 layer id; the small schedule's ids are its +positions):: + + tokens [S] int64, logits [vocab] fp32 (last position) + meta: layer_ids, compress_ratio{id}, window, kv_sources, index_sources, candidate_source, seq_len + blocks[id]: x_in [S,4,D] (after Engram), pre_in [S,4] fp32, attn_in [S,D] (normed), attn_out [S,D], + ffn_in [S,D] (normed), ffn_out [S,D], x_out [S,4,D], pre_out [S,4] (next block's pre_mix), + window_kv [S,head_dim] (post-RoPE, post-FP8-QDQ window KV of every position); + Engram layers also engram_in [S,4,D] and engram_hash_ids [S,n_hash_cols] + shared[id] (as published by that source's attention): + KV sources: compress_kv [S//r,head_dim] (RoPE + FP4 QDQ), index_k [S//r,index_head_dim] + index sources: topk_idxs [S,min(index_topk,S//r)] int32 compressed row ids (window offset + removed), -1 = unused slot + candidate source: candidates [S,S//r] bool + state (after the prompt): window{id} [window,head_dim] ring (slot = pos % window); + carry{id} {kv, score} [ratio,head_dim] fp32 (ratio-2 KV sources; slots >= S % ratio are + 0 / -inf); engram_tokens [S] compressed token ids (Engram schedules); + main_hidden [S,n_taps*D] and dspark_window{k} [window,head_dim] (DSpark schedules) + +Disk cache (``CACHE_DIR``, env ``TT_V41_ORACLE_CACHE``). Results are keyed by the model args, layer ids, +seed, weights source (``synthetic``, the checkpoint revision, or the revision + checkpoint units + ``synthetic`` +for a mixed spec), tokenizer source, a sha256 of the prompt +tokens, a digest of the vendored reference sources (model/kernel_cpu/engram/testing .py), the torch +version and ``PACKAGE_VERSION``; synthetic weights per unit by unit name, parameter signature, seed and +the same digest. Invalidation: editing a vendored reference file or changing torch invalidates +automatically; bump ``PACKAGE_VERSION`` whenever this module changes what is captured or how weights are +derived; delete the directory to reclaim space (files are ``oracle-*.pt`` and ``weights-*.pt``). +""" + +from __future__ import annotations + +import hashlib +import json +import os +import time +from dataclasses import asdict, dataclass, replace +from pathlib import Path + +import numpy as np +import torch + +from models.common import timing_events +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.engram import EngramLayout, compute_hash_multipliers +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.testing import ( + StubTokenizer, + init_weights, + prefill, + quantize_fp8_rows, + small_model_args, +) +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig, V41BlockType + +PACKAGE_VERSION = 1 +CACHE_DIR = Path(os.environ.get("TT_V41_ORACLE_CACHE", Path.home() / ".cache" / "tt-v41-oracle")) +HF_SNAPSHOT = Path( + os.environ.get( + "TT_V41_HF_SNAPSHOT", + Path.home() + / ".cache/huggingface/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots" + / "dba1be0a40aa45a94ad051997016db3960a90277", + ) +) +_HERE = Path(__file__).parent +_REFERENCE_SOURCES = ("model.py", "kernel_cpu.py", "engram.py", "testing.py") +# per-token captures, sliced by chunk_expectations +_TOKEN_KEYS = ( + "x_in", + "pre_in", + "attn_in", + "attn_out", + "ffn_in", + "ffn_out", + "x_out", + "pre_out", + "window_kv", + "engram_in", + "engram_hash_ids", +) + + +@dataclass(frozen=True) +class OracleSpec: + """What determines a reference model: its args, the checkpoint layer id of each backbone position, + the full model those layers come from (Engram hashing depends on it), the seed of synthetic weights, + the checkpoint snapshot (None = synthetic weights) and the units read from it (empty = all units; else only + these, the others synthetic).""" + + args: v41.ModelArgs + layer_ids: tuple[int, ...] + model_args: v41.ModelArgs + seed: int = 0 + checkpoint: Path | None = None + checkpoint_units: tuple[str, ...] = () + + def __post_init__(self): + if self.checkpoint_units and self.checkpoint is None: + raise ValueError(f"checkpoint_units {self.checkpoint_units} need a checkpoint") + + def unit_checkpoint(self, name: str) -> Path | None: + """The checkpoint unit ``name`` is read from, None if it is synthetic.""" + if self.checkpoint is None or (self.checkpoint_units and name not in self.checkpoint_units): + return None + return self.checkpoint + + @property + def has_synthetic(self) -> bool: + """Whether any unit has synthetic weights (then results depend on the synthetic init).""" + return self.checkpoint is None or bool(self.checkpoint_units) + + +def released_args(seq_len: int) -> v41.ModelArgs: + """The released V4.1-Flash args (vendored ``config.json``), text only, batch 1, greedy.""" + cfg = { + k: tuple(v) if isinstance(v, list) else v for k, v in json.loads((_HERE / "config.json").read_text()).items() + } + return replace(v41.ModelArgs(**cfg), max_batch_size=1, max_seq_len=seq_len, temperature=0.0, vision_n_layers=0) + + +def real_spec( + layers: tuple[int, ...], + seq_len: int, + *, + dspark: bool = False, + candidate_topk_blocks: int | None = None, + seed: int = 0, + checkpoint: Path | None = None, + checkpoint_units: tuple[str, ...] = (), +) -> OracleSpec: + """A reference made of V4.1 layers ``layers`` (execution order) at real dims. + + Every layer keeps its V4.1 role: compress ratio (0 = sliding window only, base RoPE), KV / index / + candidate source, Engram (layers 1, 14). A layer that reads shared state needs the layers producing + it in ``layers``; ``dspark`` adds the 3 DSpark layers and needs their tap layers 37-39. + ``checkpoint_units``: read only these units from ``checkpoint`` (see ``OracleSpec``). + ``candidate_topk_blocks`` overrides 2048 (at short prompts every block is a candidate otherwise). + """ + cfg = DeepSeekV41FlashConfig + layers = tuple(layers) + if not layers or list(layers) != sorted(set(layers)) or not 0 <= layers[0] <= layers[-1] < cfg.NUM_LAYERS: + raise ValueError(f"layers must be strictly increasing V4.1 backbone layer ids, got {layers}") + needed = set() + for layer in layers: + if cfg.compress_ratio(layer): + needed |= {cfg.kv_source(layer), cfg.index_source(layer)} + if cfg.block_type(layer) == V41BlockType.CANDIDATE_INDEX_SOURCE: + needed.add(cfg.CANDIDATE_SOURCE_LAYER) + if dspark: + needed |= set(cfg.DSPARK_TARGET_LAYER_IDS) + if missing := sorted(needed - set(layers)): + raise ValueError(f"layers {layers} read shared state or taps of layers {missing}, which must be included") + + full = released_args(seq_len) + pos = {layer: i for i, layer in enumerate(layers)} + + def positions(ids): + return tuple(pos[i] for i in ids if i in pos) + + engram = [i for i, layer in enumerate(full.engram_layer_ids) if layer in pos] + args = replace( + full, + n_layers=len(layers), + n_mtp_layers=cfg.NUM_DSPARK_LAYERS if dspark else 0, + compress_ratios=tuple(cfg.compress_ratio(i) for i in layers) + (0,) * (cfg.NUM_DSPARK_LAYERS if dspark else 0), + kv_source_layers=positions(full.kv_source_layers), + index_source_layers=positions(full.index_source_layers), + candidate_source_layer=pos.get(full.candidate_source_layer, -1), + candidate_topk_blocks=candidate_topk_blocks or full.candidate_topk_blocks, + engram_layer_ids=tuple(pos[full.engram_layer_ids[i]] for i in engram), + engram_num_embeddings=tuple(full.engram_num_embeddings[i] for i in engram), + dspark_block_size=full.dspark_block_size if dspark else 0, + dspark_target_layer_ids=positions(full.dspark_target_layer_ids) if dspark else (), + ) + return OracleSpec( + args, layers, full, seed, None if checkpoint is None else Path(checkpoint), tuple(checkpoint_units) + ) + + +def small_spec(seq_len: int, *, seed: int = 0, **overrides) -> OracleSpec: + """The small-dims schedule of ``testing.small_model_args`` (six block types, Engram, one DSpark + layer); layer ids are positions. Synthetic weights only.""" + args = small_model_args(max_seq_len=seq_len, **overrides) + return OracleSpec(args, tuple(range(args.n_layers)), args, seed) + + +def random_tokens(spec: OracleSpec, seq_len: int | None = None, seed: int = 1) -> torch.Tensor: + """[1, S] uniformly random token ids (S defaults to the spec's max_seq_len).""" + seq_len = seq_len or spec.args.max_seq_len + return torch.randint(0, spec.args.vocab_size, (1, seq_len), generator=torch.Generator().manual_seed(seed)) + + +def text_tokens(seq_len: int) -> torch.Tensor: + """[1, seq_len] V4.1-tokenized real text: A Tale of Two Cities from its first chapter (the book the + tt_transformers accuracy references use), for accuracy gates that score predictions of real text.""" + import bz2 + + from transformers import AutoTokenizer + + book = bz2.open(_HERE.parents[3] / "tt_transformers" / "tests" / "tale-of-two-cities.txt.bz2", "rt").read() + text = book[book.index("It was the best of times") :][: 12 * seq_len] # > 1 character per token + ids = AutoTokenizer.from_pretrained(HF_SNAPSHOT)(text, add_special_tokens=False)["input_ids"] + if len(ids) < seq_len: + raise ValueError(f"{len(ids)} tokens of text, {seq_len} requested") + return torch.tensor(ids[:seq_len], dtype=torch.int64)[None] + + +# ---------------------------------------------------------------------------------------------- model + + +def _digest(*parts) -> str: + return hashlib.sha256(json.dumps(parts, sort_keys=True, default=str).encode()).hexdigest()[:20] + + +def _reference_digest(synthetic: bool = True) -> str: + """Digest of the reference sources a result depends on. ``testing.py`` only builds synthetic weights, so + results from checkpoint weights exclude it (a synthetic-init change must not invalidate real results).""" + h = hashlib.sha256() + for name in _REFERENCE_SOURCES: + if synthetic or name != "testing.py": + h.update((_HERE / name).read_bytes()) + return h.hexdigest()[:16] + + +def _uses_real_tokenizer(spec: OracleSpec) -> bool: + return spec.model_args.vocab_size == DeepSeekV41FlashConfig.VOCAB_SIZE + + +def _tokenizer(spec: OracleSpec): + if not spec.args.engram_layer_ids: + return None + if not _uses_real_tokenizer(spec): + return StubTokenizer(spec.args.vocab_size) + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(spec.checkpoint or HF_SNAPSHOT) + + +def _source_ids(spec: OracleSpec) -> dict: + """The weights and tokenizer identities that enter every cache key.""" + weights = "synthetic" if spec.checkpoint is None else f"hf:{spec.checkpoint.name}" + if spec.checkpoint_units: # only mixed specs extend the key: synthetic / checkpoint keys stay as they were + weights += f"[{','.join(sorted(spec.checkpoint_units))}]+synthetic" + tokenizer = f"hf:{(spec.checkpoint or HF_SNAPSHOT).name}" if _uses_real_tokenizer(spec) else "stub" + return {"weights": weights, "tokenizer": tokenizer} + + +def _units(model: v41.Transformer, spec: OracleSpec) -> list[tuple[str, torch.nn.Module]]: + """(checkpoint name, module) of every weight unit, backbone layers under their checkpoint ids.""" + units = [("embed", model.embed)] + units += [(f"layers.{spec.layer_ids[i]}", layer) for i, layer in enumerate(model.layers)] + units += [("norm", model.norm), ("head", model.head)] + units += [(f"mtp.{k}", blk) for k, blk in enumerate(model.mtp)] + return units + + +class _Detached: + """Within the block, hide the shared embed/head that the DSpark blocks register as submodules.""" + + def __init__(self, module: torch.nn.Module): + self.module, self.saved = module, {} + + def __enter__(self) -> torch.nn.Module: + for name in ("embed", "head"): + if isinstance(getattr(self.module, name, None), torch.nn.Module): + self.saved[name] = getattr(self.module, name) + setattr(self.module, name, None) + return self.module + + def __exit__(self, *exc): + for name, mod in self.saved.items(): + setattr(self.module, name, mod) + + +def _unit_seed(seed: int, name: str) -> int: + # 32 bits: torch's CPU generator only uses the low 32 bits of a seed + return int(hashlib.sha256(f"{seed}:{name}".encode()).hexdigest()[:8], 16) + + +def _signature(module: torch.nn.Module) -> list: + return [(n, tuple(p.shape), str(p.dtype)) for n, p in module.named_parameters()] + + +def _attach_scales(module: torch.nn.Module) -> None: + # load_state_dict(assign=True) drops the `.scale` attribute Linear attaches to its weight + for mod in module.modules(): + if isinstance(mod, v41.Linear) and mod.scale is not None: + mod.weight.scale = mod.scale + + +def _synthetic_unit(name: str, module: torch.nn.Module, seed: int) -> None: + path = CACHE_DIR / f"weights-{_digest(name, _signature(module), seed, _reference_digest(), PACKAGE_VERSION)}.pt" + if path.is_file(): + module.load_state_dict(torch.load(path, mmap=True), assign=True) + _attach_scales(module) + return + # a top-level unit is wrapped under its name, so init_weights sees e.g. "norm" as a norm owner + init_weights(module if "." in name else torch.nn.ModuleDict({name: module}), _unit_seed(seed, name)) + CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + torch.save(module.state_dict(), tmp) + tmp.replace(path) + + +def _checkpoint_unit(name: str, module: torch.nn.Module, checkpoint: Path) -> None: + """Load a unit from the HF shards, converted as upstream convert.py does for one rank.""" + from safetensors import safe_open + + if name.startswith("layers.") and getattr(module, "engram", None) is not None: + raise NotImplementedError(f"{name}: checkpoint Engram tables (384M rows) are not loaded by the oracle") + weight_map = json.loads((checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + prefix = name + "." + by_file: dict[str, list[str]] = {} + for full_name, file in weight_map.items(): + if full_name.startswith(prefix): + by_file.setdefault(file, []).append(full_name) + if not by_file: + raise KeyError(f"no tensors named {prefix}* in {checkpoint}") + if missing := sorted(f for f in by_file if not (checkpoint / f).is_file()): + raise FileNotFoundError(f"{name} needs shards {missing}, which are not in {checkpoint}") + sd = {} + for file, names in by_file.items(): + with safe_open(checkpoint / file, framework="pt") as f: + for full_name in names: + sd[full_name[len(prefix) :]] = f.get_tensor(full_name) + for key in [k for k in sd if k.endswith("wo_a.weight")]: + w, s = sd[key], sd.pop(key.replace("weight", "scale")) + bo, bi = w.size(0) // s.size(0), w.size(1) // s.size(1) + w = w.unflatten(0, (-1, bo)).unflatten(-1, (-1, bi)).float() * s[:, None, :, None].float() + sd[key] = w.flatten(2, 3).flatten(0, 1).bfloat16() + for key in sd: + if ".experts." in key and "shared" not in key and sd[key].dtype == torch.int8: + sd[key] = sd[key].view(torch.float4_e2m1fn_x2) + if name.startswith("mtp."): + sd.pop("embed.weight", None), sd.pop("head.weight", None) # tied to the backbone's (convert.py) + if getattr(getattr(module, "ffn", None), "gate", None) is not None and module.ffn.gate.bias_vl is None: + sd.pop("ffn.gate.bias_vl", None) # the VL routing bias only acts on image tokens (text-only oracle) + module.load_state_dict(sd, strict=True) # copy_: converts e.g. bf16 head -> fp32 like load_model + + +def synthetic_engram_rows( + seed: int, layer_id: int, rows: torch.Tensor, head_dim: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Rows ``rows`` of the synthetic Engram table of checkpoint layer ``layer_id``, in the table format + (float8_e4m3fn [n, head_dim], E8M0 [n, head_dim/32]). Each row is N(0, 1) (as testing.init_weights + draws tables) from its own generator, so it depends only on (seed, layer, row).""" + w = torch.empty(len(rows), head_dim, dtype=torch.float32) # independent of the caller's default dtype + for i, r in enumerate(rows.tolist()): + row_seed = _unit_seed(seed, f"layers.{layer_id}.engram.embed.{r}") + torch.randn(head_dim, generator=torch.Generator().manual_seed(row_seed), out=w[i]) + return quantize_fp8_rows(w) + + +def _make_engram_row_sparse(engram: v41.Engram) -> None: + """Replace the table by an empty one; ``load_engram_rows`` fills the rows a prompt needs, and a + pre-hook maps global hash ids onto them before the reference lookup runs.""" + emb = engram.embed + emb.weight = torch.nn.Parameter(torch.empty(0, emb.dim, dtype=torch.float8_e4m3fn), requires_grad=False) + emb.scale = torch.nn.Parameter( + torch.empty(0, emb.dim // emb.block_size, dtype=v41.scale_dtype), requires_grad=False + ) + emb.oracle_rows = torch.empty(0, dtype=torch.int64) + + def remap(mod, args): + ids = args[0] + local = torch.searchsorted(mod.oracle_rows, ids.contiguous()).clamp_max(max(len(mod.oracle_rows) - 1, 0)) + if len(mod.oracle_rows) == 0 or not torch.equal(mod.oracle_rows[local], ids): + raise RuntimeError("Engram rows were not loaded for this prompt (call load_engram_rows first)") + return (local,) + + emb.register_forward_pre_hook(remap) + + +def load_engram_rows(model: v41.Transformer, spec: OracleSpec, tokens: torch.Tensor) -> None: + """Materialize the synthetic Engram rows ``tokens`` hash to (before the prefill runs).""" + if model.engram_hash is None: + return + with torch.inference_mode(): + hashes = model.engram_hash(tokens, 0) # [B, S, n_engram_layers, n_hash_cols], deterministic + for pos, layer in enumerate(model.layers): + if layer.engram is None: + continue + emb = layer.engram.embed + rows = torch.unique(hashes[:, :, layer.engram.layer_hash_index].flatten().clone()) + q, s = synthetic_engram_rows(spec.seed, spec.layer_ids[pos], rows, emb.dim) + emb.weight = torch.nn.Parameter(q, requires_grad=False) + emb.scale = torch.nn.Parameter(s, requires_grad=False) + emb.vocab_end_idx = len(rows) + emb.oracle_rows = rows + + +def _align_engram_hash(hash_state: v41.NgramHashState, spec: OracleSpec) -> None: + """Give a schedule's Engram layers the hash of their checkpoint layers: primes, offsets and + multipliers come from the full model's layout, selected for this schedule's Engram layers.""" + full = EngramLayout.from_args(spec.model_args) + local = [spec.layer_ids[p] for p in spec.args.engram_layer_ids] + sel = [full.layer_ids.index(layer) for layer in local] + flat = [[p for per_ngram in layer for p in per_ngram] for layer in full.primes] + offsets = np.array([np.cumsum([0, *sizes[:-1]]) for sizes in flat]) + multipliers = compute_hash_multipliers( + full.layer_ids, full.max_ngram_size, spec.model_args.engram_compressed_vocab_size + ) + hash_state.primes = torch.tensor(full.primes)[sel] + hash_state.offsets = torch.tensor(offsets)[sel] + hash_state.multipliers = multipliers[sel] + + +# queries per indexer score block: [block, index_n_heads, S/ratio] bf16 = 1.8 GB at S=56320 (the unblocked score is +# [S, index_n_heads, S/ratio], 200 GB at S=56320 ratio 1, plus as much again for the weighted product) +INDEXER_QUERY_BLOCK = 512 + + +@torch.no_grad() +def _indexer_forward_blocked(self: v41.Indexer, x, qr, latent, start_pos: int, offset: int): + """``v41.Indexer.forward`` with the score, visibility mask, candidate selection and top-k run per block of + ``INDEXER_QUERY_BLOCK`` queries, so host memory grows with S * S/ratio instead of S * heads * S/ratio. Every step + after the index keys is row-local (a query's score row, its mask, its candidate blocks and its top-k depend only + on that query), and no score's summation order changes: the per-(query, head, key) dot product is the same GEMM + element and the head reduction the same sum over dim 2. Bit-identical to ``v41.Indexer.forward`` + (tests/v41/torch/test_v41_oracle.py); keep the two in step. Installed on every reference ``build_reference`` + makes.""" + assert self.freqs_cis is not None + assert v41.world_size == 1, "the oracle runs on one rank (no index-score all-reduce)" + bsz, seqlen, _ = x.size() + ratio, rd, end_pos = self.compress_ratio, self.rope_head_dim, start_pos + seqlen + shared = v41.shared_attn + + if self.owns_k and latent is not None: # as v41.Indexer.forward + freqs = ( + self.freqs_cis[: seqlen - seqlen % ratio : ratio] + if start_pos == 0 + else self.freqs_cis[start_pos + 1 - ratio].unsqueeze(0) + ) + k = self.k_norm(self.wk(latent)) + v41.apply_rotary_emb(k[..., -rd:], freqs) + v41.fp4_act_quant(k, v41.fp4_block_size, True) + self.k_cache[:bsz, start_pos // ratio : start_pos // ratio + k.size(1)] = k + shared.index_k = self.k_cache + + q = self.wq_b(qr).unflatten(-1, (self.n_local_heads, self.index_head_dim)) + v41.apply_rotary_emb(q[..., -rd:], self.freqs_cis[start_pos:end_pos]) + v41.fp4_act_quant(q, v41.fp4_block_size, True) + + index_k = shared.index_k[:bsz, : end_pos // ratio] + weights = self.weights_proj(x) * (self.softmax_scale * self.n_heads**-0.5) + if start_pos == 0: + compress_lens = (torch.arange(1, seqlen + 1, device=x.device) // ratio).unsqueeze(-1) + else: + compress_lens = end_pos // ratio + topk = min(self.index_topk, end_pos // ratio) + candidates, idxs = [], [] + for s0 in range(0, seqlen, INDEXER_QUERY_BLOCK): + s1 = min(s0 + INDEXER_QUERY_BLOCK, seqlen) + lens = compress_lens[s0:s1] if start_pos == 0 else compress_lens + score = torch.einsum("bshd,btd->bsht", q[:, s0:s1], index_k) + score = (score.relu_() * weights[:, s0:s1].unsqueeze(-1)).sum(dim=2) + if start_pos == 0: + score.masked_fill_(torch.arange(seqlen // ratio, device=x.device) >= lens, -torch.inf) + if self.is_candidate_source: + candidates.append( + v41.select_candidate_blocks(score, lens, self.candidate_topk_blocks, self.candidate_block_size) + ) + elif self.uses_candidates: + score = score.masked_fill(~shared.candidates[:, s0:s1], -torch.inf) + block = score.topk(topk, dim=-1, sorted=False).indices.sort(dim=-1).values + idxs.append(torch.where(block < lens, block + offset, -1).int()) + del score + if self.is_candidate_source: + shared.candidates = torch.cat(candidates, dim=1) + return torch.cat(idxs, dim=1) + + +def _install_blocked_indexer(model: v41.Transformer) -> None: + for mod in model.modules(): + if isinstance(mod, v41.Indexer): + mod.forward = _indexer_forward_blocked.__get__(mod) + + +def build_reference(spec: OracleSpec) -> v41.Transformer: + """The reference model for ``spec`` with synthetic (seeded, disk-cached) or checkpoint weights. Its indexers + score in query blocks (``_indexer_forward_blocked``, bit-identical to the reference's).""" + with v41.set_dtype(torch.bfloat16): + model = v41.Transformer(spec.args, _tokenizer(spec)) + if model.engram_hash is not None: + _align_engram_hash(model.engram_hash, spec) + for layer in model.layers: + if layer.engram is not None: + _make_engram_row_sparse(layer.engram) + units = _units(model, spec) + if unknown := sorted(set(spec.checkpoint_units) - {name for name, _ in units}): + raise ValueError(f"checkpoint_units {unknown} are not units of this model") + for name, module in units: + with _Detached(module) as unit: + if (checkpoint := spec.unit_checkpoint(name)) is None: + _synthetic_unit(name, unit, spec.seed) + else: + _checkpoint_unit(name, unit, checkpoint) + _install_blocked_indexer(model) + return model.eval() + + +class LazyReference: + """The reference model for ``spec``, built on first use only: with warm oracle and weight caches a test never + builds it. Pass it wherever a ``model`` is accepted; ``built`` tells whether it was needed.""" + + def __init__(self, spec: OracleSpec): + self.spec, self._model = spec, None + + @property + def built(self) -> bool: + return self._model is not None + + def __call__(self) -> v41.Transformer: + if self._model is None: + start = time.perf_counter() + with timing_events.phase("reference", layers=list(self.spec.layer_ids)): + self._model = build_reference(self.spec) + print( + f"oracle: reference built for layers {list(self.spec.layer_ids)} in {time.perf_counter() - start:.1f}s" + ) + return self._model + + +def _model(model, spec: OracleSpec) -> v41.Transformer: + """``model`` (a reference or a LazyReference) resolved, else a freshly built reference.""" + if isinstance(model, LazyReference): + return model() + return model if model is not None else build_reference(spec) + + +# --------------------------------------------------------------------------------------------- oracle + + +def _reset_state(model: v41.Transformer) -> None: + """Fresh-request state, so ring slots / carries a prompt does not write are defined (0, -inf).""" + for mod in model.modules(): + for name in ("window_kv_cache", "compress_kv_cache", "kv_state"): + if isinstance(getattr(mod, name, None), torch.Tensor): + getattr(mod, name).zero_() + if isinstance(mod, v41.Compressor) and hasattr(mod, "score_state"): + mod.score_state.fill_(-torch.inf) + if isinstance(mod, v41.Indexer) and mod.owns_k: + mod.k_cache.zero_() + v41.shared_attn.__init__() + + +def _meta(spec: OracleSpec) -> dict: + a, ids = spec.args, spec.layer_ids + return { + "layer_ids": list(ids), + "compress_ratio": {ids[p]: a.compress_ratios[p] for p in range(a.n_layers)}, + "window": a.window_size, + "kv_sources": [ids[p] for p in a.kv_source_layers], + "index_sources": [ids[p] for p in a.index_source_layers], + "candidate_source": ids[a.candidate_source_layer] if a.candidate_source_layer >= 0 else None, + "engram_layers": [ids[p] for p in a.engram_layer_ids], + } + + +@torch.no_grad() +def _run(model: v41.Transformer, spec: OracleSpec, tokens: torch.Tensor) -> dict: + """Single-shot prefill of ``tokens`` [1, S] with block / shared-state / final-state captures.""" + seq = tokens.size(1) + _reset_state(model) + if spec.has_synthetic: # checkpoint Engram layers are rejected by _checkpoint_unit + load_engram_rows(model, spec, tokens) + blocks: dict = {lid: {} for lid in spec.layer_ids} + shared: dict = {} + hooks = [] + + def add(mod, pre=None, post=None): + if pre: + hooks.append(mod.register_forward_pre_hook(pre)) + if post: + hooks.append(mod.register_forward_hook(post)) + + for pos, layer in enumerate(model.layers): + lid, attn, rec = spec.layer_ids[pos], layer.attn, blocks[spec.layer_ids[pos]] + stash = {} + + def block_pre(mod, args, rec=rec): + rec["x_in"], rec["pre_in"] = args[0][0].clone(), args[2][0].clone() + + def block_post(mod, args, out, rec=rec): + rec["x_out"], rec["pre_out"] = out[0][0].clone(), out[1][0].clone() + + def attn_pre(mod, args, rec=rec): + rec["attn_in"] = args[0][0].clone() + + def kv_norm_post(mod, args, out, stash=stash): + stash["kv"] = out # RoPE and FP8 QDQ then run in place on this tensor + + def attn_post(mod, args, out, rec=rec, stash=stash, lid=lid): + rec["attn_out"] = out[0].clone() + rec["window_kv"] = stash.pop("kv")[0].clone() + ratio, pub = mod.compress_ratio, {} + if mod.is_kv_source: + pub["compress_kv"] = mod.compress_kv_cache[0, : seq // ratio].clone() + pub["index_k"] = mod.indexer.k_cache[0, : seq // ratio].clone() + if mod.is_index_source: + raw = v41.shared_attn.topk_idxs[0] + pub["topk_idxs"] = torch.where(raw >= 0, raw - seq, -1).int() # window offset = S + if mod.indexer.is_candidate_source: + pub["candidates"] = v41.shared_attn.candidates[0].clone() + if pub: + shared[lid] = pub + + def ffn_pre(mod, args, rec=rec): + rec["ffn_in"] = args[0][0].clone() + + def ffn_post(mod, args, out, rec=rec): + rec["ffn_out"] = out[0].clone() + + def engram_pre(mod, args, rec=rec): + rec["engram_in"], rec["engram_hash_ids"] = args[0][0].clone(), args[1][0].clone() + + add(layer, block_pre, block_post) + add(attn, attn_pre, attn_post) + add(attn.kv_norm, post=kv_norm_post) + add(layer.ffn, ffn_pre, ffn_post) + if layer.engram is not None: + add(layer.engram, pre=engram_pre) + try: + logits, main_hidden = prefill(model, tokens) + finally: + for h in hooks: + h.remove() + + state: dict = {"window": {}, "carry": {}} + for pos, layer in enumerate(model.layers): + lid, attn = spec.layer_ids[pos], layer.attn + state["window"][lid] = attn.window_kv_cache[0].clone() + if attn.compressor is not None and attn.compress_ratio > 1: + state["carry"][lid] = { + "kv": attn.compressor.kv_state[0].clone(), + "score": attn.compressor.score_state[0].clone(), + } + if model.engram_hash is not None: + state["engram_tokens"] = model.engram_hash.cache[0, :seq].clone() + if len(model.mtp): + state["main_hidden"] = main_hidden[0].clone() + state["dspark_window"] = {k: blk.attn.window_kv_cache[0].clone() for k, blk in enumerate(model.mtp)} + return { + "tokens": tokens[0].clone(), + "logits": logits[0].float(), + "meta": _meta(spec) | {"seq_len": seq}, + "blocks": blocks, + "shared": shared, + "state": state, + } + + +def cache_path(spec: OracleSpec, tokens: torch.Tensor) -> Path: + """Disk location of the result for (spec, tokens); see the module docstring for the key.""" + key = _digest( + asdict(spec.args), + asdict(spec.model_args), + list(spec.layer_ids), + spec.seed, + _source_ids(spec), + hashlib.sha256(tokens.to(torch.int64).contiguous().numpy().tobytes()).hexdigest(), + list(tokens.shape), + _reference_digest(synthetic=spec.has_synthetic), + torch.__version__, + PACKAGE_VERSION, + ) + return CACHE_DIR / f"oracle-{key}.pt" + + +def oracle(spec: OracleSpec, tokens: torch.Tensor, model: v41.Transformer | LazyReference | None = None) -> dict: + """Single-shot expected results for ``tokens`` [1, S] (S <= spec.args.max_seq_len), from the disk + cache or computed (with ``model``, else a freshly built reference) and stored.""" + if tokens.dim() != 2 or tokens.size(0) != 1 or not 0 < tokens.size(1) <= spec.args.max_seq_len: + raise ValueError(f"tokens must be [1, S] with 0 < S <= {spec.args.max_seq_len}, got {list(tokens.shape)}") + path = cache_path(spec, tokens) + if path.is_file(): + timing_events.cache(True, "oracle", path.stem, path.stat().st_size) + return torch.load(path, mmap=True) # pages in what the caller reads (tens of GB at long prompts) + timing_events.cache(False, "oracle", path.stem) + model = _model(model, spec) + with timing_events.phase("oracle", key=path.stem): + result = _run(model, spec, tokens) + CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + torch.save(result, tmp) + tmp.replace(path) + return result + + +@torch.no_grad() +def tail_logits( + spec: OracleSpec, + tokens: torch.Tensor, + count: int, + model: v41.Transformer | LazyReference | None = None, + noise: tuple[float, float, int] | None = None, +) -> torch.Tensor: + """fp32 logits ``[count, vocab]`` of the last ``count`` positions of a single-shot prefill of ``tokens`` + [1, S]: a prompt is teacher-forced by construction, so these are next-token predictions at ``count`` + positions. ``noise = (output_rel, input_rel, seed)`` adds Gaussian noise of RMS ``output_rel`` x the output's + RMS to the output of every backbone attention and MoE (component-level error) and of RMS ``input_rel`` x the + input's RMS to every attention and MoE input (the input-level sensitivity that flips top-k selection and + expert routing): the floor an implementation whose components meet their bars is gated against. Cached next + to the ``oracle`` result of the same (spec, tokens), keyed additionally by ``count`` and ``noise``.""" + path = _noisy_path(spec, tokens, count, noise, "tail") + if path.is_file(): + timing_events.cache(True, "oracle.tail", path.stem, path.stat().st_size) + return torch.load(path) + timing_events.cache(False, "oracle.tail", path.stem) + return _noisy_run(spec, tokens, count, model, noise)[0] + + +@torch.no_grad() +def noise_drift( + spec: OracleSpec, + tokens: torch.Tensor, + count: int, + noise: tuple[float, float, int], + model: v41.Transformer | LazyReference | None = None, + chunk: int | None = None, +) -> dict: + """{layer id: {"all": pcc, "tail": pcc}}: how far each block output of the prefill under ``noise`` (as + ``tail_logits``) drifts from the clean ``oracle`` result, over all rows and over the last ``count`` rows: the + per-layer floor a free-running implementation's streams are gated against. ``chunk`` adds ``"chunk"``: the drift + over the last ``chunk`` rows (the last chunk of a chunked prefill). Cached like ``tail_logits``.""" + path = _noisy_path(spec, tokens, count, noise, "drift", chunk) + if path.is_file(): + timing_events.cache(True, "oracle.drift", path.stem, path.stat().st_size) + return torch.load(path) + timing_events.cache(False, "oracle.drift", path.stem) + return _noisy_run(spec, tokens, count, model, noise, chunk)[1] + + +def _noisy_path(spec: OracleSpec, tokens: torch.Tensor, count: int, noise, kind: str, chunk: int | None = None) -> Path: + base = cache_path(spec, tokens) + tag = f"-tail{count}" if kind == "tail" else f"-drift{count}" + (f"-chunk{chunk}" if chunk else "") + tag += f"-noise-out{noise[0]:g}-in{noise[1]:g}-{noise[2]}" if noise else "" + return base.with_name(base.stem + tag + ".pt") + + +def _pcc(a: torch.Tensor, b: torch.Tensor) -> float: + return torch.corrcoef(torch.stack([a.double().flatten(), b.double().flatten()]))[0, 1].item() + + +def _save(obj, path: Path) -> None: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + torch.save(obj, tmp) + tmp.replace(path) + + +def _noisy_run( + spec: OracleSpec, tokens: torch.Tensor, count: int, model, noise, chunk: int | None = None +) -> tuple[torch.Tensor, dict | None]: + """One prefill (with ``noise`` if given): stores and returns the tail logits and, with noise, the drift.""" + if not 0 < count <= tokens.size(1): + raise ValueError(f"count must be in (0, {tokens.size(1)}], got {count}") + if chunk is not None and not 0 < chunk <= tokens.size(1): + raise ValueError(f"chunk must be in (0, {tokens.size(1)}], got {chunk}") + model = _model(model, spec) + clean = oracle(spec, tokens, model) if noise is not None else None # before the noisy run resets the state + _reset_state(model) + if spec.has_synthetic: # checkpoint Engram layers are rejected by _checkpoint_unit + load_engram_rows(model, spec, tokens) + captured, blocks = {}, {} + + def capture(mod, args): # the backbone's head call comes first (DSpark calls the head again afterwards) + captured.setdefault("x", args[0][0, -count:].clone()) + + hooks = [model.head.register_forward_pre_hook(capture)] + if noise is not None: + output_rel, input_rel, seed = noise + gen = torch.Generator().manual_seed(seed) + + def perturbed(t, rel): + return (t.float() + torch.randn(t.shape, generator=gen) * rel * t.float().pow(2).mean().sqrt()).to(t.dtype) + + def perturb_output(mod, args, out): + return perturbed(out, output_rel) + + def perturb_input(mod, args): + return (perturbed(args[0], input_rel), *args[1:]) + + for pos, layer in enumerate(model.layers): + for m in (layer.attn, layer.ffn): + hooks += [m.register_forward_pre_hook(perturb_input), m.register_forward_hook(perturb_output)] + + def block_out(mod, args, out, lid=spec.layer_ids[pos]): + blocks[lid] = out[0][0].clone() + + hooks.append(layer.register_forward_hook(block_out)) + try: + with timing_events.phase("oracle", key=_noisy_path(spec, tokens, count, noise, "tail").stem): + prefill(model, tokens) + finally: + for h in hooks: + h.remove() + # one row at a time, as the reference head projects the last position (bit-identical to its logits) + weight = model.head.weight.float() + logits = torch.cat([torch.nn.functional.linear(row[None].float(), weight) for row in captured["x"]]) + _save(logits, _noisy_path(spec, tokens, count, noise, "tail")) + if noise is None: + return logits, None + drift = {} + for lid, x in blocks.items(): + ref = clean["blocks"][lid]["x_out"] + drift[lid] = {"all": _pcc(ref, x), "tail": _pcc(ref[-count:], x[-count:])} + if chunk: + drift[lid]["chunk"] = _pcc(ref[-chunk:], x[-chunk:]) + _save(drift, _noisy_path(spec, tokens, count, noise, "drift", chunk)) + return logits, drift + + +# ------------------------------------------------------------------------------------ chunk contract + + +def window_ring(window_kv: torch.Tensor, end: int, window: int) -> torch.Tensor: + """The window ring after positions [0, end): slot p % window holds position p's window KV for the + last ``window`` positions; slots never written stay 0 (graph.md §4 rule 1).""" + ring = window_kv.new_zeros(window, window_kv.size(-1)) + pos = torch.arange(max(0, end - window), end) + ring[pos % window] = window_kv[pos] + return ring + + +def chunk_expectations(result: dict, chunk_len: int, pad_multiple: int) -> list[dict]: + """What a chunked device prefill must match, per chunk, given the single-shot ``result``. + + Chunk c covers valid tokens [start, start + length) and runs padded to ``padded_length`` (a + multiple of ``pad_multiple``, e.g. 2*32*SP); positions >= length write nothing (rule 8). Per chunk: + ``blocks[id]`` per-token captures sliced to the valid tokens; ``rows[src]`` the compressed-KV and + index-K rows [row_start, row_end) = [start//r, (start+length)//r) that the chunk writes, and nothing + beyond row_end; ``topk_idxs[id]`` / ``candidates[id]`` rows of the chunk's queries (candidates over + the row_end visible columns; top-k: compare the sets of valid ids, the single-shot width is + min(topk, S//r)); ``window[id]`` the ring after the chunk; ``engram_history`` the compressed ids of + the last 3 real tokens. The last chunk adds ``logits`` and the final ``state`` (carries, rings, + DSpark), which equal the single-shot ones. + """ + meta, seq = result["meta"], result["meta"]["seq_len"] + ratios = [r for r in meta["compress_ratio"].values() if r] + if pad_multiple <= 0 or chunk_len % pad_multiple or any(pad_multiple % r for r in ratios): + raise ValueError( + f"chunk_len ({chunk_len}) must be a multiple of pad_multiple ({pad_multiple}), itself a multiple " + f"of every compress ratio {sorted(set(ratios))}" + ) + chunks = [] + for index, start in enumerate(range(0, seq, chunk_len)): + end = min(start + chunk_len, seq) + length = end - start + c = { + "index": index, + "start": start, + "length": length, + "padded_length": -(-length // pad_multiple) * pad_multiple, + "blocks": { + lid: {k: v[start:end] for k, v in rec.items() if k in _TOKEN_KEYS} + for lid, rec in result["blocks"].items() + }, + "rows": {}, + "topk_idxs": {}, + "candidates": {}, + "window": { + lid: window_ring(rec["window_kv"], end, meta["window"]) for lid, rec in result["blocks"].items() + }, + } + for lid, pub in result["shared"].items(): + r = meta["compress_ratio"][lid] + if "compress_kv" in pub: + lo, hi = start // r, end // r + c["rows"][lid] = { + "row_start": lo, + "row_end": hi, + "compress_kv": pub["compress_kv"][lo:hi], + "index_k": pub["index_k"][lo:hi], + } + if "topk_idxs" in pub: + c["topk_idxs"][lid] = pub["topk_idxs"][start:end] + if "candidates" in pub: + c["candidates"][lid] = pub["candidates"][start:end, : end // r] + if "engram_tokens" in result["state"]: + c["engram_history"] = result["state"]["engram_tokens"][max(0, end - 3) : end] + if end == seq: + c["logits"], c["state"] = result["logits"], result["state"] + chunks.append(c) + return chunks + + +# ----------------------------------------------------------------------------------- vision (ViT + aligner) +# +# ``vision_oracle`` runs the reference image encoder (``Transformer.encode_image``: ``vision.ViT`` then +# ``vision.Aligner``) on one image's patches. Weights are synthetic (per-tensor seeded) or the checkpoint's +# ``vision.*`` / ``aligner.*`` tensors (shard 1, stored bf16, used as stored). Results are cached as +# ``vision-*.pt``, keyed by the vision args, weights source, a sha256 of the patches, the grid, a digest of +# ``vision.py``, the torch version and ``PACKAGE_VERSION``. + +_VISION_SOURCES = ("vision.py",) + + +def vision_args() -> v41.ModelArgs: + """The released V4.1-Flash args (vendored ``config.json``) with the vision tower enabled.""" + cfg = { + k: tuple(v) if isinstance(v, list) else v for k, v in json.loads((_HERE / "config.json").read_text()).items() + } + return v41.ModelArgs(**cfg) + + +def _vision_modules(args: v41.ModelArgs): + from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import vision + + with v41.set_dtype(torch.bfloat16): # as upstream builds the model (norm weights stay fp32) + return vision.ViT(args), vision.Aligner(args) + + +def vision_weight_shapes(args: v41.ModelArgs) -> dict[str, torch.Size]: + """Checkpoint name -> shape of every ViT / aligner tensor, from the reference modules.""" + vit, aligner = _vision_modules(args) + return {f"{p}{k}": v.shape for p, m in (("vision.", vit), ("aligner.", aligner)) for k, v in m.state_dict().items()} + + +def synthetic_vision_weights(args: v41.ModelArgs, seed: int = 0) -> dict[str, torch.Tensor]: + """bf16 ViT / aligner weights by checkpoint name: norms 1 + 0.1 N(0,1), biases 0.02 N(0,1), matrices + N(0,1) / sqrt(fan_in). Each tensor is drawn from its own seed (seed, name), so it does not depend on + the other tensors or on ``vision_n_layers``.""" + out = {} + for name, shape in vision_weight_shapes(args).items(): + t = torch.randn(shape, generator=torch.Generator().manual_seed(_unit_seed(seed, name)), dtype=torch.float32) + if name.endswith("norm1.weight") or name.endswith("norm2.weight") or name.endswith("norm.weight"): + t = 1 + 0.1 * t + elif name.endswith(".bias"): + t = 0.02 * t + else: + t = t * shape[-1] ** -0.5 + out[name] = t.to(torch.bfloat16) + return out + + +def checkpoint_vision_weights(checkpoint: Path = HF_SNAPSHOT) -> dict[str, torch.Tensor]: + """The checkpoint's ViT / aligner tensors by name, as stored.""" + from safetensors import safe_open + + names = vision_weight_shapes(vision_args()) + weight_map = json.loads((Path(checkpoint) / "model.safetensors.index.json").read_text())["weight_map"] + if missing := sorted(n for n in names if n not in weight_map): + raise KeyError(f"{checkpoint} has no tensors {missing[:3]}...") + by_file: dict[str, list[str]] = {} + for name in names: + by_file.setdefault(weight_map[name], []).append(name) + if absent := sorted(f for f in by_file if not (Path(checkpoint) / f).is_file()): + raise FileNotFoundError(f"vision weights need shards {absent}, which are not in {checkpoint}") + out = {} + for file, file_names in by_file.items(): + with safe_open(Path(checkpoint) / file, framework="pt") as f: + out |= {n: f.get_tensor(n) for n in file_names} + return out + + +def build_vision_reference(args: v41.ModelArgs, weights: dict[str, torch.Tensor]): + """Reference ``(ViT, Aligner)`` holding ``weights`` (by checkpoint name), converted to the modules' + dtypes as upstream ``load_state_dict`` does.""" + vit, aligner = _vision_modules(args) + for prefix, module in (("vision.", vit), ("aligner.", aligner)): + module.load_state_dict({k: weights[prefix + k].to(v.dtype) for k, v in module.state_dict().items()}) + return vit.eval(), aligner.eval() + + +def synthetic_image(width: int, height: int, seed: int = 0) -> bytes: + """A deterministic PNG with structure (colour gradients, a disc) plus mild noise.""" + import io + + from PIL import Image + + y, x = np.mgrid[0:height, 0:width].astype(np.float32) + disc = np.hypot(x - width * 0.4, y - height * 0.6) < min(width, height) * 0.25 + img = np.stack([255 * x / width, 255 * y / height, 128 + 100 * disc], axis=-1) + img += torch.randn(img.shape, generator=torch.Generator().manual_seed(seed)).numpy() * 8 + buf = io.BytesIO() + Image.fromarray(np.clip(img, 0, 255).astype(np.uint8)).save(buf, format="PNG") + return buf.getvalue() + + +def image_patches(image: bytes, args: v41.ModelArgs | None = None) -> tuple[torch.Tensor, int, int]: + """``(patches [n_h*n_w, 3, p, p] bf16, n_h, n_w)`` of an encoded image, via ``image_processor.load_image``.""" + from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import image_processor + + patches, n_h, n_w, _, _ = image_processor.load_image({"data": image}, args or vision_args()) + return patches, n_h, n_w + + +def random_patches(n_h: int, n_w: int, args: v41.ModelArgs | None = None, seed: int = 0) -> torch.Tensor: + """``[n_h*n_w, 3, p, p]`` bf16 N(0, 1) patches of a direct ``n_h x n_w`` grid.""" + p = (args or vision_args()).vision_patch_size + return torch.randn(n_h * n_w, 3, p, p, generator=torch.Generator().manual_seed(seed)).to(torch.bfloat16) + + +def _vision_digest() -> str: + h = hashlib.sha256() + for name in _VISION_SOURCES: + h.update((_HERE / name).read_bytes()) + return h.hexdigest()[:16] + + +def vision_cache_path( + args: v41.ModelArgs, patches: torch.Tensor, n_h: int, n_w: int, seed: int, checkpoint: Path | None +) -> Path: + fields = {k: v for k, v in asdict(args).items() if k.startswith("vision_") or k == "dim"} + key = _digest( + "vision", + fields, + "synthetic" if checkpoint is None else f"hf:{Path(checkpoint).name}", + seed if checkpoint is None else None, + hashlib.sha256(patches.contiguous().view(torch.uint8).numpy().tobytes()).hexdigest(), + [str(patches.dtype), *patches.shape], + n_h, + n_w, + _vision_digest(), + torch.__version__, + PACKAGE_VERSION, + ) + return CACHE_DIR / f"vision-{key}.pt" + + +@torch.no_grad() +def vision_oracle( + patches: torch.Tensor, + n_h: int, + n_w: int, + *, + args: v41.ModelArgs | None = None, + seed: int = 0, + checkpoint: Path | None = None, +) -> dict: + """Expected image encoder outputs for ``patches`` [n_h*n_w, 3, p, p] of an ``n_h x n_w`` grid, from the + disk cache or computed and stored: ``hidden`` [n_h*n_w, vision_dim] bf16 (ViT output, row-major patch + order), ``aligned`` [ceil(n_h/r)*ceil(n_w/r), dim] bf16 (aligner output, reading order; the grid is + zero-padded to multiples of r), ``meta``. Weights are the synthetic ones of ``seed`` or, with + ``checkpoint``, the stored ones.""" + args = args or vision_args() + if patches.dim() != 4 or patches.size(0) != n_h * n_w or n_h <= 0 or n_w <= 0: + raise ValueError(f"patches must be [n_h*n_w = {n_h * n_w}, 3, p, p], got {list(patches.shape)}") + path = vision_cache_path(args, patches, n_h, n_w, seed, checkpoint) + if path.is_file(): + return torch.load(path) + weights = synthetic_vision_weights(args, seed) if checkpoint is None else checkpoint_vision_weights(checkpoint) + vit, aligner = build_vision_reference(args, weights) + with v41.set_dtype(torch.bfloat16): + hidden = vit(patches, n_h, n_w) + aligned = aligner(hidden, n_h, n_w) + r = args.vision_downsample_ratio + result = { + "hidden": hidden.clone(), + "aligned": aligned.clone(), + "meta": {"n_h": n_h, "n_w": n_w, "n_llm_h": -(-n_h // r), "n_llm_w": -(-n_w // r)}, + } + CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + torch.save(result, tmp) + tmp.replace(path) + return result + + +# ------------------------------------------------------------------------------ merged (VL) prompts +# Bead 10.2: the text reference run on a prompt with image spans (``Transformer.forward`` with ``images`` / +# ``token_types``). Image features are teacher-forced: the aligner rows of each span are given (e.g. a +# ``vision_oracle`` result), so a merged-sequence result depends on the backbone only. The VL reference is the +# text reference of the same spec (identical text weights, synthetic or checkpoint) plus the VL-only parameters: +# each layer's gate ``bias_vl`` and the ``image_start`` / ``image_end`` / ``image_newline`` span embeddings +# (synthetic: seeded per name; checkpoint: stored). The prompt enters every cache key of ``oracle`` / +# ``tail_logits`` / ``noise_drift`` through ``asdict(spec.args)`` (``VLArgs.vl_prompt``), so text-only keys are +# unchanged. Bump ``VL_VERSION`` whenever the VL parameter derivation changes. + +VL_VERSION = 1 +_VL_SOURCES = ("image_processor.py",) +VL_FEATURE_STD = 0.04 # RMS of real aligner rows (checkpoint, 640x480 image: 0.041) +VL_DELIMITER_STD = 0.025 # RMS of the checkpoint's image_start / image_end / image_newline (0.021-0.028) +VL_BIAS_STD = 0.5 # synthetic bias_vl, drawn like testing.init_weights draws the text gate bias + + +@dataclass +class VLArgs(v41.ModelArgs): + """ModelArgs of a merged-sequence spec: ``vl_prompt`` (``VLPrompt.digest``) keys every result by the prompt's + image spans; the model built from it is the text model (``vision_n_layers`` stays 0, no ViT).""" + + vl_prompt: str = "" + + +@dataclass(frozen=True, eq=False) +class VLPrompt: + """The image side of a merged prompt: ``token_types`` [S] int64 (``image_processor`` TEXT = -1, IMAGE_START, + IMAGE, IMAGE_NEW_LINE, IMAGE_END) and ``features``: per image span in prompt order its aligner rows + ``[T_i, dim]`` bf16, T_i = the span's IMAGE slots (reading order).""" + + token_types: torch.Tensor + features: tuple[torch.Tensor, ...] + + def spans(self) -> list[tuple[int, int]]: + """(start, end) of each image span, end exclusive, in prompt order.""" + from models.demos.deepseek_v3_d_p.reference.deepseek_v41.image_processor import IMAGE_END, IMAGE_START + + starts = (self.token_types == IMAGE_START).nonzero().flatten().tolist() + ends = (self.token_types == IMAGE_END).nonzero().flatten().tolist() + if len(starts) != len(ends) or any(e < s for s, e in zip(starts, ends)): + raise ValueError("image spans must be IMAGE_START ... IMAGE_END") + return [(s, e + 1) for s, e in zip(starts, ends)] + + def digest(self) -> str: + h = hashlib.sha256(self.token_types.to(torch.int64).contiguous().numpy().tobytes()) + for f in self.features: + h.update(str(list(f.shape)).encode()) + h.update(f.to(torch.bfloat16).contiguous().view(torch.uint16).numpy().tobytes()) + for name in _VL_SOURCES: + h.update((_HERE / name).read_bytes()) + h.update(f"vl{VL_VERSION}".encode()) + return h.hexdigest()[:20] + + +def merged_prompt( + tokens: torch.Tensor, spans: list[tuple[int, int, int, torch.Tensor]], args: v41.ModelArgs | None = None +) -> tuple[torch.Tensor, VLPrompt]: + """Overwrite text ``tokens`` [1, S] with image spans ``(start, n_llm_h, n_llm_w, features [h*w, dim])``, laid out + as ``image_processor.prepare_vl_inputs`` does (``image_token_id`` at every span position, the default + ``image_token_types`` layout). Returns (tokens [1, S], VLPrompt).""" + from models.demos.deepseek_v3_d_p.reference.deepseek_v41.image_processor import TEXT, image_token_types + + image_token_id = (args or v41.ModelArgs()).image_token_id + tokens, types = tokens.clone(), torch.full((tokens.size(1),), TEXT, dtype=torch.int64) + features = [] + for start, n_h, n_w, feats in sorted(spans, key=lambda s: s[0]): + span = image_token_types(n_h, n_w) + if feats.shape[0] != n_h * n_w: + raise ValueError(f"{feats.shape[0]} feature rows for a {n_h}x{n_w} token grid") + if start + span.numel() > tokens.size(1) or (types[start : start + span.numel()] != TEXT).any(): + raise ValueError(f"image span at {start} ({span.numel()} tokens) overlaps or leaves the prompt") + tokens[0, start : start + span.numel()] = image_token_id + types[start : start + span.numel()] = span + features.append(feats.to(torch.bfloat16)) + return tokens, VLPrompt(types, tuple(features)) + + +def synthetic_image_features(n_rows: int, dim: int, seed: int = 0) -> torch.Tensor: + """``[n_rows, dim]`` bf16 N(0, VL_FEATURE_STD^2) stand-in aligner rows.""" + g = torch.Generator().manual_seed(_unit_seed(seed, f"image_features.{n_rows}x{dim}")) + return (torch.randn(n_rows, dim, generator=g) * VL_FEATURE_STD).to(torch.bfloat16) + + +def vl_spec(spec: OracleSpec, prompt: VLPrompt) -> OracleSpec: + """``spec`` for the merged prompt: the same model, results keyed additionally by the prompt's image spans.""" + fields = {k: getattr(spec.args, k) for k in v41.ModelArgs.__dataclass_fields__} + return replace(spec, args=VLArgs(**fields, vl_prompt=prompt.digest())) + + +def vl_parameters(spec: OracleSpec) -> dict[str, torch.Tensor]: + """The VL-only parameters by checkpoint name: ``layers..ffn.gate.bias_vl`` (fp32) of every backbone layer + and ``image_start`` / ``image_end`` / ``image_newline`` (bf16 ``[dim]``); synthetic (seeded per name) or stored.""" + names = [f"layers.{lid}.ffn.gate.bias_vl" for lid in spec.layer_ids] + names += ["image_start", "image_end", "image_newline"] + if spec.checkpoint_units: + raise NotImplementedError("VL parameters of a mixed synthetic / checkpoint spec are not defined") + if spec.checkpoint is None: + out = {} + for name in names: + g = torch.Generator().manual_seed(_unit_seed(spec.seed, f"vl{VL_VERSION}:{name}")) + if name.startswith("layers."): + out[name] = VL_BIAS_STD * torch.randn(spec.args.n_routed_experts, generator=g) + else: + out[name] = (VL_DELIMITER_STD * torch.randn(spec.args.dim, generator=g)).to(torch.bfloat16) + return out + from safetensors import safe_open + + weight_map = json.loads((spec.checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + out = {} + for name in names: + with safe_open(spec.checkpoint / weight_map[name], framework="pt") as f: + out[name] = f.get_tensor(name) + return out + + +def build_vl_reference(spec: OracleSpec, prompt: VLPrompt) -> v41.Transformer: + """The reference for a merged prompt (``spec`` from ``vl_spec``): ``build_reference`` plus the VL parameters, + with ``forward`` bound to the prompt's images / token types (the IMAGE slots take ``prompt.features``: the + encoder is teacher-forced) and the Engram hash to its image mask, so ``oracle`` / ``tail_logits`` / + ``noise_drift`` run the merged prefill unchanged.""" + import functools + + from models.demos.deepseek_v3_d_p.reference.deepseek_v41.image_processor import ImageInput + + if not isinstance(spec.args, VLArgs) or spec.args.vl_prompt != prompt.digest(): + raise ValueError("spec is not vl_spec(..., prompt) for this prompt") + spans = prompt.spans() + if len(spans) != len(prompt.features): + raise ValueError(f"{len(spans)} image spans, {len(prompt.features)} feature sets") + model = build_reference(spec) + params = vl_parameters(spec) + for pos, layer in enumerate(model.layers): + bias_vl = params[f"layers.{spec.layer_ids[pos]}.ffn.gate.bias_vl"].float() + layer.ffn.gate.bias_vl = torch.nn.Parameter(bias_vl, requires_grad=False) + for name in ("image_start", "image_end", "image_newline"): + setattr(model, name, torch.nn.Parameter(params[name].to(torch.bfloat16), requires_grad=False)) + images = [[ImageInput(s, f, 0, 0, prompt.token_types[s:e]) for (s, e), f in zip(spans, prompt.features)]] + model.encode_image = lambda features, n_vit_h, n_vit_w: features # teacher-forced aligner rows + token_types = prompt.token_types[None] + model.forward = functools.partial(type(model).forward, model, images=images, token_types=token_types) + if model.engram_hash is not None: + hash_state, text = model.engram_hash, token_types < 0 + + def masked_hash(input_ids, start_pos, token_mask=None): + mask = text[:, start_pos : start_pos + input_ids.size(1)] if token_mask is None else token_mask + return type(hash_state).forward(hash_state, input_ids, start_pos, mask) + + hash_state.forward = masked_hash + return model diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/testing.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/testing.py new file mode 100644 index 000000000000..e3f0bd2b2581 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/testing.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Small, randomly initialized DeepSeek-V4.1 reference models for CPU tests. + +``small_model_args`` builds a ``ModelArgs`` whose 6 backbone layers cover the six V4.1 block types +(schedule in ``SmallScheduleConfig``, typed by the canonical ``DeepSeekV41FlashConfig.block_type``), +plus Engram on an SWA and a KV-source layer and one DSpark layer. ``build_small_model`` constructs the +vendored ``Transformer`` with a stub tokenizer (the Engram hash needs a compressed token map) and fills +every parameter deterministically from a seed, in its checkpoint storage dtype: FP8 e4m3 weights with +E8M0 32x32 block scales, FP4 (float4_e2m1fn_x2) routed experts with E8M0 per-32 scales, FP8 Engram +tables with E8M0 per-32 scales, bf16/fp32 elsewhere. +""" + +import math +from dataclasses import replace + +import torch + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.engram import EngramLayout +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.kernel_cpu import FP8_MAX, fast_round_scale +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig + +EMBED_STD = 0.12 # RMS of the released V4.1 embedding (measured on the checkpoint, layer-0 input) + +# Synthetic weights follow the released checkpoint's statistics (measured on layers 0, 2, 20; user decision +# 2026-09-29): with unit norm weights and fan-in-scaled projections the MoE output swamps a small residual +# and the first block of a synthetic stack becomes far more sensitive to tiny errors than the real model. +# Norm weight means by owner (std 10% of the mean); projection std as a multiple of fan_in^-0.5 by name. +NORM_MEAN = { + "attn_norm": 0.025, + "ffn_norm": 0.2, + "q_norm": 0.97, + "kv_norm": 0.4, + "compressor.norm": 0.6, + "k_norm": 0.5, +} +PROJECTION_GAIN = { + "wq_a": 1.6, + "wq_b": 0.95, + "wkv": 1.8, + "wo_a": 1.6, + "wo_b": 1.9, + "gate": 2.9, + "w1": 1.5, + "w3": 1.5, + "w2": 1.05, +} + + +def _norm_mean(owner_name: str) -> float: + for key, mean in NORM_MEAN.items(): + if owner_name == key or owner_name.endswith("." + key): + return mean + return 1.0 + + +def _gain(owner_name: str) -> float: + return PROJECTION_GAIN.get(owner_name.rsplit(".", 1)[-1], 1.0) + + +class SmallScheduleConfig(DeepSeekV41FlashConfig): + """The V4.1 layer-role rules on a 6-layer schedule: one layer of each block type, in the order + SWA_ONLY, KV_INDEX_SOURCE (ratio 2), CONSUMER_RATIO2, CANDIDATE_SOURCE (ratio 1), CONSUMER_RATIO1, + CANDIDATE_INDEX_SOURCE; then one ratio-0 DSpark layer.""" + + NUM_LAYERS = 6 + NUM_DSPARK_LAYERS = 1 + COMPRESS_RATIOS = (0, 2, 2, 1, 1, 1, 0) + KV_SOURCE_LAYERS = (1, 3) + INDEX_SOURCE_LAYERS = (1, 3, 5) + CANDIDATE_SOURCE_LAYER = 3 + ENGRAM_LAYER_IDS = (0, 1) + DSPARK_TARGET_LAYER_IDS = (4, 5) + + +class StubTokenizer: + """Just enough of a HF tokenizer for ``engram.build_compressed_token_map``: token 2k decodes to + "w{k}" and 2k+1 to " W{k}", which normalize alike, so the compressed vocab is vocab_size // 2.""" + + def __init__(self, vocab_size: int): + self.vocab_size = vocab_size + self.backend_tokenizer = self + + def __len__(self) -> int: + return self.vocab_size + + def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: + (token_id,) = ids + return f"w{token_id // 2}" if token_id % 2 == 0 else f" W{token_id // 2}" + + def id_to_token(self, token_id: int) -> str: + return self.decode([token_id]) + + +def small_model_args(**overrides) -> v41.ModelArgs: + """ModelArgs for a small model; attention/indexer/engram sizes keep every quantized dim a multiple of 32.""" + cfg = SmallScheduleConfig + vocab_size = 512 + args = v41.ModelArgs( + max_batch_size=1, + max_seq_len=128, + temperature=0.0, + vocab_size=vocab_size, + dim=256, + moe_inter_dim=128, + n_layers=cfg.NUM_LAYERS, + n_mtp_layers=cfg.NUM_DSPARK_LAYERS, + n_heads=4, + n_routed_experts=4, + n_activated_experts=2, + score_func=cfg.SCORE_FUNC, + route_scale=cfg.ROUTE_SCALE, + swiglu_limit=cfg.SWIGLU_LIMIT, + q_lora_rank=64, + head_dim=64, + rope_head_dim=16, + norm_eps=cfg.RMS_NORM_EPS, + o_groups=2, + o_lora_rank=64, + window_size=16, + compress_ratios=cfg.COMPRESS_RATIOS, + kv_source_layers=cfg.KV_SOURCE_LAYERS, + index_source_layers=cfg.INDEX_SOURCE_LAYERS, + compress_rope_theta=cfg.COMPRESS_ROPE_THETA, + original_seq_len=32, # < max_seq_len so the compressed layers' YaRN ramp is active + rope_theta=cfg.ROPE_THETA, + rope_factor=4, + beta_fast=cfg.ROPE_SCALING_BETA_FAST, + beta_slow=cfg.ROPE_SCALING_BETA_SLOW, + index_n_heads=4, + index_head_dim=64, + index_topk=8, + candidate_source_layer=cfg.CANDIDATE_SOURCE_LAYER, + candidate_topk_blocks=3, # 3 blocks x 4 >= index_topk, so reachable candidates always fill top-k + candidate_block_size=4, + hc_mult=cfg.HC_MULT, + hc_sinkhorn_iters=cfg.HC_SINKHORN_ITERS, + hc_eps=cfg.HC_EPS, + engram_layer_ids=cfg.ENGRAM_LAYER_IDS, + engram_max_ngram_size=cfg.ENGRAM_MAX_NGRAM_SIZE, + engram_vocab_size=100, + engram_n_heads=2, + engram_head_dim=32, + engram_pad_id=cfg.ENGRAM_PAD_ID, + engram_compressed_vocab_size=vocab_size // 2, + dspark_block_size=2, + dspark_noise_token_id=vocab_size - 1, + dspark_target_layer_ids=cfg.DSPARK_TARGET_LAYER_IDS, + dspark_markov_rank=32, + ) + args = replace(args, **overrides) + if args.engram_layer_ids and not args.engram_num_embeddings: + # table rows = the summed prime bucket ranges, as in the released config + layout = EngramLayout.from_args(replace(args, engram_num_embeddings=(0,) * len(args.engram_layer_ids))) + rows = tuple(sum(p for per_ngram in layer for p in per_ngram) for layer in layout.primes) + args = replace(args, engram_num_embeddings=rows) + return args + + +def quantize_fp8_blocks(w: torch.Tensor, block: int = 32) -> tuple[torch.Tensor, torch.Tensor]: + """fp32 [R, C] -> (float8_e4m3fn [R, C], E8M0 [ceil(R/block), C/block]) with one power-of-2 scale per + block x block tile (the checkpoint's dense FP8 format). Rows are padded to a block multiple.""" + rows, cols = w.shape + pad = -rows % block + wp = torch.cat([w, w.new_zeros(pad, cols)]) if pad else w + tiles = wp.unflatten(0, (-1, block)).unflatten(-1, (-1, block)) # [Rb, block, Cb, block] + amax = tiles.abs().amax(dim=(1, 3)).clamp_min(1e-4) + s = fast_round_scale(amax, torch.tensor(1.0 / FP8_MAX, dtype=torch.float32)) + q = (tiles / s[:, None, :, None]).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q.flatten(2, 3).flatten(0, 1)[:rows].contiguous(), s.to(torch.float8_e8m0fnu) + + +def quantize_fp8_rows(w: torch.Tensor, block: int = 32) -> tuple[torch.Tensor, torch.Tensor]: + """fp32 [R, C] -> (float8_e4m3fn [R, C], E8M0 [R, C/block]): per-row groups (Engram table format).""" + groups = w.unflatten(-1, (-1, block)) + s = fast_round_scale(groups.abs().amax(dim=-1).clamp_min(1e-4), torch.tensor(1.0 / FP8_MAX)) + q = (groups / s.unsqueeze(-1)).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn) + return q.flatten(-2), s.to(torch.float8_e8m0fnu) + + +@torch.no_grad() +def init_weights(model: torch.nn.Module, seed: int = 0) -> None: + """Fill every parameter from a seeded generator, in named_parameters order, in its storage dtype.""" + gen = torch.Generator().manual_seed(seed) + + def randn(*shape: int) -> torch.Tensor: + return torch.randn(*shape, generator=gen, dtype=torch.float32) + + modules = dict(model.named_modules()) + for name, p in model.named_parameters(): + owner_name, _, leaf = name.rpartition(".") + owner = modules[owner_name] + if p.dtype == torch.float8_e8m0fnu: + continue # written together with its weight below + if p.dtype == torch.float8_e4m3fn: + w = randn(*p.shape) + if isinstance(owner, v41.ParallelEngramEmbedding): + q, s = quantize_fp8_rows(w, owner.block_size) + else: + q, s = quantize_fp8_blocks(w * p.size(1) ** -0.5 * _gain(owner_name)) + p.copy_(q) + owner.scale.copy_(s) + elif p.dtype == torch.float4_e2m1fn_x2: + # Random E2M1 codes (both nibbles) with one power-of-2 scale per 32 inputs, drawn directly in + # the checkpoint format: quantizing a gaussian costs minutes per layer at real dims. Uniform + # codes have RMS ~3.2, so the scale targets an RMS of fan_in^-0.5 like the other weights. + fan_in = 2 * p.size(1) + codes = torch.randint(0, 256, p.shape, generator=gen, dtype=torch.int32).to(torch.uint8) + p.view(torch.uint8).copy_(codes) + exp = round(math.log2(fan_in**-0.5 * _gain(owner_name) / 3.2)) + jitter = torch.randint(-1, 2, owner.scale.shape, generator=gen, dtype=torch.int32) + owner.scale.copy_(torch.pow(2.0, (exp + jitter).float()).to(torch.float8_e8m0fnu)) + elif leaf == "weight" and "norm" in owner_name.rsplit(".", 1)[-1]: + mean = _norm_mean(owner_name) + p.copy_(mean * (1 + 0.1 * randn(*p.shape))) + elif isinstance(owner, v41.ParallelEmbedding): + # the released embedding has RMS ~0.12 (real layer-0 input); fan-in scaling would give ~0.014 and + # make the first block of a synthetic stack far more sensitive than the real model + p.copy_(randn(*p.shape) * EMBED_STD) + elif p.dim() >= 2: + p.copy_(randn(*p.shape) * p.size(-1) ** -0.5 * _gain(owner_name)) + else: + p.copy_(0.5 * randn(*p.shape)) + + +def build_small_model(args: v41.ModelArgs | None = None, seed: int = 0) -> v41.Transformer: + """Construct the vendored Transformer (bf16 default dtype, as generate.py does) and seed its weights.""" + args = args or small_model_args() + with v41.set_dtype(torch.bfloat16): + model = v41.Transformer(args, StubTokenizer(args.vocab_size)) + init_weights(model, seed) + return model.eval() + + +def prefill(model: v41.Transformer, input_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor | None]: + """One single-shot prefill (start_pos 0) under bf16 default dtype, as generate.py runs it: the + backbone forward, then the DSpark prefill that seeds the DSpark window caches. Returns full-vocab + logits of the last position [b, vocab] (fp32) and the DSpark main hidden (None without DSpark).""" + with v41.set_dtype(torch.bfloat16): + output_ids, logits, main_hidden = model(input_ids, 0) + if len(model.mtp): + assert model.forward_spec(output_ids, main_hidden, 0) is None + return logits, main_hidden diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41/vision.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/vision.py new file mode 100644 index 000000000000..40246c61cd18 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41/vision.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: © 2023 DeepSeek +# SPDX-License-Identifier: MIT +# Vendored from huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash, revision +# dba1be0a40aa45a94ad051997016db3960a90277, file inference/vision.py (MIT License). +# Deviations from upstream are listed in README.md next to this file. + +from functools import lru_cache + +import torch +import torch.nn.functional as F +from torch import nn + + +@lru_cache(8) +def get_vision_cos_sin(n_h: int, n_w: int, dim: int, theta: float): + inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + hpos = torch.arange(n_h).unsqueeze(1).expand(n_h, n_w) + wpos = torch.arange(n_w).unsqueeze(0).expand(n_h, n_w) + freqs = torch.stack([hpos, wpos], dim=-1).reshape(-1, 2, 1).float() * inv_freq + freqs = freqs.flatten(1) + return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1) + + +def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + dtype = x.dtype + x1, x2 = x.float().chunk(2, dim=-1) + return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(dtype) + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + dtype = x.dtype + x = x.float() + x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps) + return (self.weight * x).to(dtype) + + +class PatchEmbed(nn.Module): + def __init__(self, args): + super().__init__() + self.proj = nn.Linear(3 * args.vision_patch_size**2, args.vision_dim) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.proj(x.flatten(1)) + + +class Attention(nn.Module): + def __init__(self, args): + super().__init__() + self.n_heads = args.vision_n_heads + self.head_dim = args.vision_dim // args.vision_n_heads + self.wqkv = nn.Linear(args.vision_dim, 3 * args.vision_dim) + self.wo = nn.Linear(args.vision_dim, args.vision_dim) + + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + n = x.size(0) + q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in self.wqkv(x).chunk(3, dim=-1)) + q = apply_rotary(q, cos, sin) + k = apply_rotary(k, cos, sin) + o = F.scaled_dot_product_attention(q.transpose(0, 1), k.transpose(0, 1), v.transpose(0, 1)) + return self.wo(o.transpose(0, 1).reshape(n, -1)) + + +class MLP(nn.Module): + def __init__(self, args): + super().__init__() + self.w1 = nn.Linear(args.vision_dim, 2 * args.vision_inter_dim, bias=False) + self.w2 = nn.Linear(args.vision_inter_dim, args.vision_dim, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate, up = self.w1(x).chunk(2, dim=-1) + return self.w2(F.silu(gate) * up) + + +class Block(nn.Module): + def __init__(self, args): + super().__init__() + self.norm1 = RMSNorm(args.vision_dim) + self.attn = Attention(args) + self.norm2 = RMSNorm(args.vision_dim) + self.mlp = MLP(args) + + def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + x = x + self.attn(self.norm1(x), cos, sin) + return x + self.mlp(self.norm2(x)) + + +class ViT(nn.Module): + """DeepSeek ViT: full bidirectional attention over one image with 2D RoPE.""" + + def __init__(self, args): + super().__init__() + self.rope_dim = args.vision_dim // args.vision_n_heads // 2 + self.rope_theta = args.vision_rope_theta + self.patch_embed = PatchEmbed(args) + self.blocks = nn.ModuleList([Block(args) for _ in range(args.vision_n_layers)]) + self.norm = RMSNorm(args.vision_dim) + + def forward(self, patches: torch.Tensor, n_h: int, n_w: int) -> torch.Tensor: + x = self.patch_embed(patches) + cos, sin = get_vision_cos_sin(n_h, n_w, self.rope_dim, self.rope_theta) + for block in self.blocks: + x = block(x, cos, sin) + return self.norm(x) + + +class Aligner(nn.Module): + def __init__(self, args): + super().__init__() + self.downsample_ratio = args.vision_downsample_ratio + in_dim = args.vision_dim * self.downsample_ratio**2 + self.w1 = nn.Linear(in_dim, args.dim) + self.w2 = nn.Linear(args.dim, args.dim) + + def forward(self, x: torch.Tensor, n_h: int, n_w: int) -> torch.Tensor: + r = self.downsample_ratio + x = x.view(n_h, n_w, -1).permute(2, 0, 1) + x = F.pad(x, (0, -n_w % r, 0, -n_h % r)) + x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).transpose(0, 1) + return self.w2(F.gelu(self.w1(x))) diff --git a/models/demos/deepseek_v3_d_p/reference/deepseek_v41_flash_config.py b/models/demos/deepseek_v3_d_p/reference/deepseek_v41_flash_config.py new file mode 100644 index 000000000000..789ca34af41a --- /dev/null +++ b/models/demos/deepseek_v3_d_p/reference/deepseek_v41_flash_config.py @@ -0,0 +1,165 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +""" +DeepSeek V4.1 Flash Model Configuration. + +Single source of truth for model dimension constants and the per-layer attention schedule. +Values from ``inference/config.json`` of HuggingFace ``deepseek-ai/DeepSeek-V4.1-Flash`` (revision +dba1be0a40aa). Layer roles are derived from the reference's source lists exactly as +``inference/model.py`` wires them: a consumer reads the compressed KV / index keys / top-k of the most +recent source executed before it (``SharedAttentionRuntime``). +""" + +from enum import Enum + + +class V41BlockType(Enum): + """The six V4.1 decoder block types, by attention role.""" + + SWA_ONLY = "swa_only" # sliding window only (layers 0, 1) + KV_INDEX_SOURCE = "kv_index_source" # compresses its KV, owns index keys, runs its indexer + CONSUMER_RATIO2 = "consumer_ratio2" # reads the latest ratio-2 source's compressed KV and top-k + CANDIDATE_SOURCE = "candidate_source" # KV + index source that also publishes candidate blocks + CANDIDATE_INDEX_SOURCE = "candidate_index_source" # own indexer, masked to the published candidates + CONSUMER_RATIO1 = "consumer_ratio1" # reads layer 20's ratio-1 KV and the latest index source's top-k + + +class DeepSeekV41FlashConfig: + """DeepSeek V4.1 Flash model dimensions and layer schedule.""" + + # Core dimensions + EMB_SIZE = 5120 + FABRIC_PAYLOAD_SIZE = EMB_SIZE # max fabric packet payload; must stay in sync with migration code + MOE_INTERMEDIATE_SIZE = 2304 + HEAD_DIM = 512 + + # MoE configuration + NUM_ROUTED_EXPERTS = 384 + NUM_EXPERTS_PER_TOKEN = 6 + NUM_SHARED_EXPERTS = 1 + NUM_EXPERT_GROUPS = 1 + NUM_LIMITED_GROUPS = 1 + SCORE_FUNC = "sqrtsoftplus" + ROUTE_SCALE = 1.5 + ROUTED_EXPERT_ACTIVATION = "clamped_silu_glu" + SHARED_EXPERT_ACTIVATION = "clamped_silu_glu" + SWIGLU_LIMIT = 10.0 + + # Model architecture + NUM_LAYERS = 40 # backbone; DSpark layers follow at indices 40..42 + NUM_DSPARK_LAYERS = 3 + NUM_DENSE_LAYERS = 0 + VOCAB_SIZE = 129280 + SLIDING_WINDOW = 128 + + # MLA dimensions + NUM_ATTENTION_HEADS = 64 + NUM_KEY_VALUE_HEADS = 1 + Q_LORA_RANK = 1280 + O_LORA_RANK = 1024 + O_GROUPS = 8 + QK_ROPE_HEAD_DIM = 64 + + # Indexer / sparse attention + INDEX_N_HEADS = 32 + INDEX_HEAD_DIM = 128 + INDEX_TOPK = 512 + CANDIDATE_TOPK_BLOCKS = 2048 + CANDIDATE_BLOCK_SIZE = 8 + + # Per-layer attention schedule, DSpark layers included (config.json compress_ratios). + COMPRESS_RATIOS = (0, 0) + (2,) * 18 + (1,) * 20 + (0,) * 3 + KV_SOURCE_LAYERS = (2, 8, 14, 20) + INDEX_SOURCE_LAYERS = (2, 8, 14, 20, 24, 28, 32, 36) + CANDIDATE_SOURCE_LAYER = 20 + + # RoPE: ratio-0 layers use ROPE_THETA without YaRN; compressed layers use COMPRESS_ROPE_THETA + YaRN. + ROPE_THETA = 10000 + COMPRESS_ROPE_THETA = 160000.0 + ROPE_SCALING_FACTOR = 16 + ROPE_SCALING_ORIGINAL_MAX_POSITION_EMBEDDINGS = 65536 + ROPE_SCALING_BETA_FAST = 32 + ROPE_SCALING_BETA_SLOW = 1 + + # Hyper-connections + HC_MULT = 4 + HC_SINKHORN_ITERS = 20 + HC_EPS = 1.0e-6 + + # Every RMSNorm, including the hc_mixes normalization (the vision tower keeps its own 1e-6). + RMS_NORM_EPS = 1e-20 + + # Engram + ENGRAM_LAYER_IDS = (1, 14) + ENGRAM_NUM_EMBEDDINGS = (384006168, 384016682) + ENGRAM_VOCAB_SIZE = 16000000 + ENGRAM_MAX_NGRAM_SIZE = 4 + ENGRAM_N_HEADS = 8 + ENGRAM_HEAD_DIM = 256 + ENGRAM_PAD_ID = 2 + ENGRAM_COMPRESSED_VOCAB_SIZE = 99092 + + # DSpark (prefill only seeds the DSpark window caches from these taps) + DSPARK_TARGET_LAYER_IDS = (37, 38, 39) + + # Vision + VISION_N_LAYERS = 32 + VISION_DIM = 1024 + VISION_N_HEADS = 16 + VISION_INTER_DIM = 2816 + VISION_PATCH_SIZE = 14 + VISION_DOWNSAMPLE_RATIO = 3 + VISION_MAX_N_TOKEN = 1024 + VISION_ROPE_THETA = 10000 + VISION_NORM_EPS = 1e-6 + IMAGE_TOKEN_ID = 129264 + + MAX_POSITION_EMBEDDINGS = 1048576 + + @classmethod + def compress_ratio(cls, layer: int) -> int: + return cls.COMPRESS_RATIOS[layer] + + @classmethod + def _latest_source(cls, layer: int, sources: tuple[int, ...]) -> int: + """The most recent source executed at or before ``layer``: the one whose shared state it reads.""" + candidates = [s for s in sources if s <= layer] + if not candidates: + raise ValueError(f"layer {layer} has no preceding source in {sources}") + return candidates[-1] + + @classmethod + def kv_source(cls, layer: int) -> int: + """The layer that produced the compressed KV and index keys ``layer`` attends over.""" + if layer >= cls.NUM_LAYERS or cls.compress_ratio(layer) == 0: + raise ValueError(f"layer {layer} does not attend over compressed KV") + source = cls._latest_source(layer, cls.KV_SOURCE_LAYERS) + if cls.compress_ratio(source) != cls.compress_ratio(layer): + raise ValueError(f"layer {layer} and its KV source {source} have different compress ratios") + return source + + @classmethod + def index_source(cls, layer: int) -> int: + """The layer whose top-k selection ``layer`` uses.""" + if layer >= cls.NUM_LAYERS or cls.compress_ratio(layer) == 0: + raise ValueError(f"layer {layer} does not use a top-k selection") + return cls._latest_source(layer, cls.INDEX_SOURCE_LAYERS) + + @classmethod + def block_type(cls, layer: int) -> V41BlockType: + if not 0 <= layer < cls.NUM_LAYERS: + raise ValueError(f"layer {layer} is not a backbone layer") + ratio = cls.compress_ratio(layer) + if ratio == 0: + return V41BlockType.SWA_ONLY + if layer == cls.CANDIDATE_SOURCE_LAYER: + return V41BlockType.CANDIDATE_SOURCE + if layer in cls.KV_SOURCE_LAYERS: + return V41BlockType.KV_INDEX_SOURCE + if layer in cls.INDEX_SOURCE_LAYERS: + if not 0 <= cls.CANDIDATE_SOURCE_LAYER < layer: + raise ValueError(f"index source {layer} owns no keys and has no candidate source before it") + return V41BlockType.CANDIDATE_INDEX_SOURCE + return V41BlockType.CONSUMER_RATIO2 if ratio == 2 else V41BlockType.CONSUMER_RATIO1 diff --git a/models/demos/deepseek_v3_d_p/tests/pcc/test_moe_gate_prefill2d.py b/models/demos/deepseek_v3_d_p/tests/pcc/test_moe_gate_prefill2d.py index 763330518aea..e678bced84fd 100644 --- a/models/demos/deepseek_v3_d_p/tests/pcc/test_moe_gate_prefill2d.py +++ b/models/demos/deepseek_v3_d_p/tests/pcc/test_moe_gate_prefill2d.py @@ -17,6 +17,7 @@ from models.demos.deepseek_v3_d_p.reference.deepseek_v3_config import DeepSeekV3Config from models.demos.deepseek_v3_d_p.reference.deepseek_v4_flash_config import DeepSeekV4FlashConfig from models.demos.deepseek_v3_d_p.reference.deepseek_v4_pro_config import DeepSeekV4ProConfig +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig from models.demos.deepseek_v3_d_p.reference.glm_5_2_config import GLM52Config from models.demos.deepseek_v3_d_p.reference.gpt_oss.modeling_gpt_oss import GptOssTopKRouter from models.demos.deepseek_v3_d_p.reference.gpt_oss_120b_config import GptOss120BConfig @@ -67,6 +68,7 @@ "gpt_oss_120b": GptOss120BConfig, "dsv4_pro": DeepSeekV4ProConfig, "dsv4_flash": DeepSeekV4FlashConfig, + "dsv41_flash": DeepSeekV41FlashConfig, "mistral_small_4": MistralSmall4Config, } @@ -269,6 +271,7 @@ def _try_load_real_gate_input(max_seq_len: int, dim: int) -> torch.Tensor | None pytest.param("gpt_oss_120b", GateComputeMode.GPT_DEVICE, id="gpt_oss_120b-gpt_device"), pytest.param("dsv4_pro", GateComputeMode.DEVICE_FP32, id="dsv4_pro-device_fp32"), pytest.param("dsv4_flash", GateComputeMode.DEVICE_FP32, id="dsv4_flash-device_fp32"), + pytest.param("dsv41_flash", GateComputeMode.DEVICE_FP32, id="dsv41_flash-device_fp32"), # Mistral's router is softmax -> top-4 -> renormalize, which is the GPT-OSS rule: softmax is # monotonic, so top-k on logits equals top-k on softmax, and renormalizing the selected k equals # softmaxing them. The sigmoid device gate cannot express it (moe_grouped_topk.cpp's diff --git a/models/demos/deepseek_v3_d_p/tests/v41/engram_reference.py b/models/demos/deepseek_v3_d_p/tests/v41/engram_reference.py new file mode 100644 index 000000000000..916c6243363d --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/engram_reference.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Cached CPU expected results of one DeepSeek-V4.1 Engram layer (bead F5), from the vendored reference. + +For a schedule whose first layer is an Engram layer (e.g. ``real_spec((1,), S)``), the Engram input is the +expanded embedding; the reference hash (``NgramHashState``, optionally with an image mask), row lookup +(``ParallelEngramEmbedding`` on the synthetic rows the prompt hashes to) and ``Engram.forward`` run once and are +cached on disk (``oracle.CACHE_DIR``). No ttnn import: run it in a plain torch process before a device run:: + + python -m models.demos.deepseek_v3_d_p.tests.v41.engram_reference + +Result: tokens [S], mask [S] bool or None, hash_ids [S, n_hash_cols] (global rows), table (weight fp8 +[R, head_dim], scale E8M0 [R, head_dim/32], row_ids [R] sorted: the rows the prompt needs), rows [S, n_hash_cols +* head_dim] bf16 (reference lookup), x_in [S, hc, D] bf16, out_fp32 [S, hc, D] (``Engram.forward`` on the fp32 +view of x_in: the update without the final bf16 rounding), out [S, hc, D] bf16 (the reference output). +""" + +from __future__ import annotations + +import hashlib +import time +from dataclasses import asdict + +import torch + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.tests.v41.small_config import small_spec + +VERSION = 2 +SMALL_SEQ, REAL_SEQ = 512, 5120 +# name -> (spec, prompt length, image span or None); layer 1 is the first Engram layer (SWA-only, no sources) +CASES = { + "small": (lambda: small_spec((1,), SMALL_SEQ), SMALL_SEQ, None), + "small_padded": (lambda: small_spec((1,), SMALL_SEQ), SMALL_SEQ - 12, None), + "small_image": (lambda: small_spec((1,), SMALL_SEQ), SMALL_SEQ, (250, 270)), # spans the 256 chunk boundary + "real": (lambda: orc.real_spec((1,), REAL_SEQ), REAL_SEQ, None), +} + + +def image_mask(seq: int, start: int, end: int) -> torch.Tensor: + """[S] bool token mask: False (image token) on [start, end).""" + mask = torch.ones(seq, dtype=torch.bool) + mask[start:end] = False + return mask + + +def case_inputs(name: str) -> tuple[orc.OracleSpec, torch.Tensor, torch.Tensor | None]: + """(spec, tokens [S], token mask [S] or None) of a named case.""" + make_spec, length, span = CASES[name] + spec = make_spec() + tokens = orc.random_tokens(spec, length)[0] + return spec, tokens, None if span is None else image_mask(length, *span) + + +def _path(spec: orc.OracleSpec, tokens: torch.Tensor, mask: torch.Tensor | None): + sha = lambda t: None if t is None else hashlib.sha256(t.to(torch.int64).contiguous().numpy().tobytes()).hexdigest() + key = orc._digest( + asdict(spec.args), + asdict(spec.model_args), + list(spec.layer_ids), + spec.seed, + sha(tokens), + sha(mask), + orc._reference_digest(), + torch.__version__, + VERSION, + ) + return orc.CACHE_DIR / f"engram-{key}.pt" + + +def _load_rows(emb, seed: int, layer_id: int, ids: torch.Tensor): + """Materialize the synthetic rows ``ids`` hash to in the reference table (as oracle.load_engram_rows).""" + rows = torch.unique(ids.flatten()) + weight, scale = orc.synthetic_engram_rows(seed, layer_id, rows, emb.dim) + emb.weight = torch.nn.Parameter(weight, requires_grad=False) + emb.scale = torch.nn.Parameter(scale, requires_grad=False) + emb.vocab_end_idx = len(rows) + emb.oracle_rows = rows + return {"weight": weight, "scale": scale, "row_ids": rows} + + +@torch.no_grad() +def engram_case( + spec: orc.OracleSpec, tokens: torch.Tensor, mask: torch.Tensor | None = None, model: v41.Transformer | None = None +) -> dict: + """Expected Engram results of the spec's first layer (an Engram layer) for ``tokens`` [S] (cached).""" + path = _path(spec, tokens, mask) + if path.is_file(): + return torch.load(path) + assert 0 in spec.args.engram_layer_ids, "the schedule's first layer must be an Engram layer" + model = model if model is not None else orc.build_reference(spec) + engram = model.layers[0].engram + started = time.time() + with torch.inference_mode(): + ids = model.engram_hash(tokens[None], 0, None if mask is None else mask[None])[0, :, engram.layer_hash_index] + table = _load_rows(engram.embed, spec.seed, spec.layer_ids[0], ids) + with v41.set_dtype(torch.bfloat16), torch.inference_mode(): + rows = engram.embed(ids).flatten(-2) + x_in = model.embed(tokens[None]).unsqueeze(2).repeat(1, 1, spec.args.hc_mult, 1) + token_mask = None if mask is None else mask[None] + out = engram(x_in, ids[None], token_mask)[0] + out_fp32 = engram(x_in.float(), ids[None], token_mask)[0] + result = { + "tokens": tokens.clone(), + "mask": None if mask is None else mask.clone(), + "hash_ids": ids.clone(), + "table": table, + "rows": rows.clone(), + "x_in": x_in[0].clone(), + "out_fp32": out_fp32.clone(), + "out": out.clone(), + "seconds": time.time() - started, + } + orc.CACHE_DIR.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + torch.save(result, tmp) + tmp.replace(path) + return result + + +if __name__ == "__main__": + # precompute the cases of tests/v41/test_engram_v41.py + import sys + + torch.set_num_threads(16) + for name in sys.argv[1:] or CASES: + started = time.time() + engram_case(*case_inputs(name)) + print(f"{name}: {time.time() - started:.1f} s") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/expert_dtype_reference.py b/models/demos/deepseek_v3_d_p/tests/v41/expert_dtype_reference.py new file mode 100644 index 000000000000..43cb6320a90a --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/expert_dtype_reference.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""CPU reference for the routed-expert dtype study (dev-spec D-B, ``test_v41_expert_dtype``); torch only, no ttnn. + +The MoE input of real layer ``layer`` is the §6 block oracle's ``ffn_in`` (the same 2048-token real-weight oracle +``test_block_v41`` uses: schedule ``stack`` for layer 2, ``swa`` for layer 0), so MoE and block numbers of this +study share one input. The observable MoE parts (``moe_reference.reference_parts``) are computed on those 2048 +rows and checked bit for bit against the oracle's ``ffn_out``; the device runs a production 5120-token chunk made +by tiling the rows (every MoE part is row-local, so the expected parts tile the same way). + +Warm the cache outside the device lock: +``python -m models.demos.deepseek_v3_d_p.tests.v41.expert_dtype_reference 2 0``. +""" + +import hashlib +import sys +import time + +import torch +from loguru import logger + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as O +from models.demos.deepseek_v3_d_p.tests.v41.moe_reference import reference_parts + +ORACLE_SEQ = 2048 +CHUNK = 5120 # production prefill chunk the device MoE runs +SCHEDULES = {2: (2, 3, 20, 21, 24), 20: (2, 3, 20, 21, 24), 0: (0,)} # test_block_v41 SCHEDULES: stack, swa +REFERENCE_VERSION = 1 + + +def block_spec(layer: int, layers: tuple[int, ...] | None = None) -> O.OracleSpec: + """``test_block_v41``'s real-weight spec of the schedule holding ``layer`` (or of ``layers``).""" + return O.real_spec(layers or SCHEDULES[layer], ORACLE_SEQ, candidate_topk_blocks=96, checkpoint=O.HF_SNAPSHOT) + + +def tile_rows(t: torch.Tensor, rows: int = CHUNK) -> torch.Tensor: + reps = -(-rows // t.shape[0]) + return t.repeat(reps, *([1] * (t.dim() - 1)))[:rows] + + +def moe_parts(layer: int) -> dict: + """Reference MoE parts of real ``layer`` on its block-oracle input (cached), tiled to ``CHUNK`` rows.""" + spec = block_spec(layer) + tokens = O.random_tokens(spec) + name = O.cache_path(spec, tokens).name + key = hashlib.sha256(f"{name}:{layer}:{REFERENCE_VERSION}".encode()).hexdigest()[:20] + path = O.CACHE_DIR / f"moe-dtype-parts-{key}.pt" + if path.is_file(): + start = time.perf_counter() + parts = torch.load(path) + logger.info(f"MoE parts layer {layer}: cached {path.name} {time.perf_counter() - start:.1f}s") + else: + start = time.perf_counter() + result = O.oracle(spec, tokens) # cached by the block tests + x = result["blocks"][layer]["ffn_in"] + logger.info(f"block oracle {name} loaded {time.perf_counter() - start:.1f}s") + start = time.perf_counter() + model = O.build_reference(block_spec(layer, (layer,))) # one real layer: only its MoE is used + logger.info(f"reference layer {layer} built {time.perf_counter() - start:.1f}s") + start = time.perf_counter() + with v41.set_dtype(torch.bfloat16): + parts = reference_parts(model.layers[0].ffn, x) + assert torch.equal(parts["final"], result["blocks"][layer]["ffn_out"]), "MoE split differs from MoE.forward" + parts["x"] = x + logger.info(f"MoE parts layer {layer} computed {time.perf_counter() - start:.1f}s") + tmp = path.with_suffix(".tmp") + torch.save(parts, tmp) + tmp.replace(path) + return {k: tile_rows(v) for k, v in parts.items()} + + +if __name__ == "__main__": + torch.set_num_threads(8) + for arg in sys.argv[1:]: + parts = moe_parts(int(arg)) + print(arg, {k: tuple(v.shape) for k, v in parts.items()}, flush=True) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/expert_load_profile.py b/models/demos/deepseek_v3_d_p/tests/v41/expert_load_profile.py new file mode 100644 index 000000000000..230ff4f392ed --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/expert_load_profile.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Generates ``tt/v41/expert_load_profile.json``, the routed-expert load profile behind the V4.1 expert placement +(``tt/v41/expert_placement.py``, beads 8y7.9.8, 8y7.9.12). CPU only (torch; no ttnn, no device):: + + python models/demos/deepseek_v3_d_p/tests/v41/expert_load_profile.py + +Per prompt and length (``SEQS``: the first 2048 and the first 5120 tokens, V4.1 tokenizer, no special tokens; a +prompt shorter than 5120 tokens gives all of them): the tokens run through the reference stack of the checkpoint +layers (0, 2, 3, 20, 21, 24) (``oracle.oracle``, disk-cached; cold ~4 min at 2048, ~4-9 min at 5120 per prompt, +8 threads); the reference gate (``Gate.forward``: fp32 ``sqrt(softplus(x W^T))``, top-6 of ``scores + bias``) on each +layer's MoE input ``ffn_in`` gives the routed pairs per expert (one profile row). Only these layers have checkpoint +shards here. The 2048-token spec keeps 96 candidate blocks (the block tests' oracles); the 5120-token spec is the +production chunk's (``test_v41_moe_placement.text_spec``). + +Training prompts (``TRAINING``) cover technical prose, code, legal text, Q&A, narrative prose and three non-English +languages. The held-out prompts (``HELD_OUT``: the novel of ``oracle.text_tokens`` that the production gates and the +G2 profile use, and the vendored reference ``model.py`` as code) are never used for the placement; the device +placement tests run on them. Random token ids are reported as a stress case (2048 tokens), not trained on. + +The ``validation`` field records the modelled MoE cost of the most expensive chip (``expert_placement.chip_costs``, +LoudBox 2x4, 5120-token chunk) for the checkpoint order and for the placement: leave-one-prompt-out over the training +prompts (both rows of the prompt left out, evaluated on its 5120-token row) and on each held-out prompt. +""" + +import hashlib +import json +import time +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F +from loguru import logger + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tt.v41 import expert_placement as P +from models.demos.deepseek_v3_d_p.tt.v41.weights import resolve_checkpoint + +REPO = Path(__file__).resolve().parents[5] +LAYERS = (0, 2, 3, 20, 21, 24) # every backbone layer with checkpoint shards present +SEQ = 2048 # the short profile row (and the default of spec / prompt_tokens) +SEQS = (SEQ, 5120) +CHIPS, GROUP = 8, 2 # LoudBox 2x4: 4 dispatch groups (mesh columns) of 2 chips +EMACS = Path("/usr/share/emacs/27.1/etc/tutorials") +LICENSES = Path("/usr/share/common-licenses") + + +def _file(path: Path): + return lambda: path.read_text() + + +def _json_prompts(path: Path): + return lambda: "\n\n".join(item["prompt"] for item in json.loads(path.read_text())) + + +TRAINING = { + "docs_llms": _file(REPO / "tech_reports/LLMs/llms.md"), + "docs_fabric": _file(REPO / "tech_reports/TT-Fabric/TT-Fabric-Architecture.md"), + "docs_distributed": _file(REPO / "tech_reports/TT-Distributed/TT-Distributed-Architecture-1219.md"), + "legal_gpl3": _file(LICENSES / "GPL-3"), + "legal_gfdl": _file(LICENSES / "GFDL-1.3"), + "code_cpp": _file(REPO / "tt_metal/impl/program/dispatch.cpp"), + "code_python": _file(REPO / "ttnn/ttnn/decorators.py"), + "qa_aime_gpqa": _json_prompts(REPO / "models/demos/deepseek_v3/demo/demo_aime24_gpqa_short.json"), + "story_25k": _json_prompts(REPO / "models/demos/deepseek_v3_d_p/demo/test_prompt_25k.json"), + "zh_tutorial": _file(EMACS / "TUTORIAL.cn"), + "ru_tutorial": _file(EMACS / "TUTORIAL.ru"), + "de_tutorial": _file(EMACS / "TUTORIAL.de"), +} +HELD_OUT = { + "novel": None, # oracle.text_tokens + "code_reference": _file(REPO / "models/demos/deepseek_v3_d_p/reference/deepseek_v41/model.py"), +} + + +def spec(seq: int = SEQ) -> orc.OracleSpec: + if seq == SEQ: + return orc.real_spec(LAYERS, SEQ, candidate_topk_blocks=96, checkpoint=orc.HF_SNAPSHOT) + return orc.real_spec(LAYERS, seq, checkpoint=orc.HF_SNAPSHOT) + + +def prompt_tokens(name: str, seq: int = SEQ) -> torch.Tensor: + """[1, <= seq] tokens of a training or held-out prompt (exactly ``SEQ`` for the short row).""" + if name == "novel": + return orc.text_tokens(seq) + if name == "random": + return orc.random_tokens(spec(seq)) + from transformers import AutoTokenizer + + text = {**TRAINING, **HELD_OUT}[name]() + ids = AutoTokenizer.from_pretrained(orc.HF_SNAPSHOT)(text, add_special_tokens=False)["input_ids"] + assert len(ids) >= SEQ, (name, len(ids)) + return torch.tensor(ids[:seq], dtype=torch.int64)[None] + + +def gate_weights(layer: int) -> tuple[torch.Tensor, torch.Tensor]: + names = [f"layers.{layer}.ffn.gate.weight", f"layers.{layer}.ffn.gate.bias"] + w = resolve_checkpoint().read(names) + return w[names[0]].float(), w[names[1]].float() + + +def reference_indices(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor) -> torch.Tensor: + """The reference ``Gate`` selection (text tokens) for the MoE inputs ``x`` [n, dim].""" + scores = F.softplus(x.float() @ weight.T).sqrt() # score_func sqrtsoftplus, gate_temp 1 + return (scores + bias).topk(C.NUM_EXPERTS_PER_TOKEN, dim=-1)[1] + + +def prompt_counts(name: str, seq: int, gates: dict) -> tuple[int, dict[int, list[int]]]: + """(tokens, routed pairs per expert per layer) of the first ``seq`` tokens of prompt ``name``.""" + tokens = prompt_tokens(name, seq) + start = time.perf_counter() + result = orc.oracle(spec(seq), tokens) + logger.info(f"{name} [{tokens.shape[1]} tokens]: oracle {time.perf_counter() - start:.1f}s") + return tokens.shape[1], { + layer: torch.bincount( + reference_indices(result["blocks"][layer]["ffn_in"], *gates[layer]).flatten(), + minlength=C.NUM_ROUTED_EXPERTS, + ).tolist() + for layer in LAYERS + } + + +def max_chip_ms(counts: np.ndarray, chip) -> float: + """Most expensive chip (ms) of the placement cost model for one prompt's routed pairs at the chunk scale.""" + return float(P.chip_costs(counts, np.asarray(chip), CHIPS).max() / 1e3) + + +def main() -> None: + torch.set_num_threads(8) + gates = {layer: gate_weights(layer) for layer in LAYERS} + rows = [(n, seq) for seq in SEQS for n in TRAINING] + counts = {(n, seq): prompt_counts(n, seq, gates) for n, seq in rows} + evaluated = {f"{n}_{seq}": prompt_counts(n, seq, gates) for n in HELD_OUT for seq in SEQS} + evaluated["random_2048"] = prompt_counts("random", SEQ, gates) + scaled = lambda tokens_counts, layer: np.asarray(tokens_counts[1][layer], dtype=np.float64) * ( + P.CHUNK_TOKENS / tokens_counts[0] + ) + identity = [e // (C.NUM_ROUTED_EXPERTS // CHIPS) for e in range(C.NUM_ROUTED_EXPERTS)] + validation = {} + for layer in LAYERS: + train = np.stack([scaled(counts[r], layer) for r in rows]) + chip = P.place(train, CHIPS) + loo = [] + for n in TRAINING: + keep = [i for i, r in enumerate(rows) if r[0] != n] + target = scaled(counts[(n, SEQS[-1])], layer) + loo.append((max_chip_ms(target, identity), max_chip_ms(target, P.place(train[keep], CHIPS)))) + row = {"leave_one_out": [round(float(np.mean([v[k] for v in loo])), 3) for k in (0, 1)]} + for n, tc in evaluated.items(): + c = scaled(tc, layer) + row[n] = [round(max_chip_ms(c, identity), 3), round(max_chip_ms(c, chip), 3)] + validation[str(layer)] = row + logger.info(f"L{layer} modelled most expensive chip ms [checkpoint order, placed]: {row}") + profile = { + "source": { + "description": "routed (token, expert) pairs per expert of the reference gate, first 2048 and first 5120 " + "tokens per prompt (one row each), reference stack of the checkpoint layers (0, 2, 3, 20, 21, 24); " + "generated by tests/v41/expert_load_profile.py", + "checkpoint": str(orc.HF_SNAPSHOT.name), + "prompts": { + n: hashlib.sha256(prompt_tokens(n, SEQS[-1]).numpy().tobytes()).hexdigest()[:16] for n in TRAINING + }, + }, + "rows": [{"prompt": n, "tokens": counts[(n, seq)][0]} for n, seq in rows], + "validation": { + "metric": f"modelled MoE cost of the most expensive chip (ms, expert_placement.chip_costs, {P.CHUNK_TOKENS}-token " + f"chunk, {GROUP}x{CHIPS // GROUP} mesh), [checkpoint order, placed]; leave_one_out: mean over the training " + "prompts of their 5120-token row, fitted without the prompt", + "held_out": { + n: hashlib.sha256(prompt_tokens(n, SEQS[-1]).numpy().tobytes()).hexdigest()[:16] for n in HELD_OUT + }, + "layers": validation, + }, + "counts": {str(layer): [counts[r][1][layer] for r in rows] for layer in LAYERS}, + } + counts_json = profile.pop("counts") + lines = ",\n".join( + f' "{layer}": [\n' + ",\n".join(f" {json.dumps(r)}" for r in layer_rows) + "\n ]" + for layer, layer_rows in counts_json.items() + ) + text = json.dumps(profile, indent=1)[:-2] + ',\n "counts": {\n' + lines + "\n }\n}\n" + assert json.loads(text)["counts"] == counts_json + tmp = P.PROFILE_PATH.with_suffix(".tmp") + tmp.write_text(text) + tmp.replace(P.PROFILE_PATH) + logger.info(f"wrote {P.PROFILE_PATH}") + + +if __name__ == "__main__": + main() diff --git a/models/demos/deepseek_v3_d_p/tests/v41/galaxy_meshes.py b/models/demos/deepseek_v3_d_p/tests/v41/galaxy_meshes.py new file mode 100644 index 000000000000..39443725712c --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/galaxy_meshes.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Galaxy mesh parameters for the V4.1 device tests (bead 8y7.13.4; spec: artifacts galaxy-acceptance.md). + +Two layouts of the 32-chip Blackhole Galaxy (SP rows x TP columns): ``torus-xy-8x4`` (production, SP8 x TP4, Ring on +both axes; the id and fabric of ``FABRIC_2D_PREFILL_BLOCK_MESH_PARAMS`` in tests/conftest.py) and ``fabric2d-mesh-4x8`` +(capacity fallback, SP4 x TP8). tests/conftest.py permits only FABRIC_1D / FABRIC_2D for a 4x8 mesh on a Galaxy +(its TorusXY descriptor check is 8x4 Ring/Ring), so 4x8 runs unwrapped Fabric2D and ``V41Collectives`` realizes +Linear on both axes. The ``requires_mesh_topology`` marks skip both on any other box (LoudBox: 8 devices). + +Galaxy chunks are multiples of 1024 (``V41MeshLayout.check_chunk``: 32 * sp * tp query split), so the small-dims +cases (512 tokens) cannot run there; tests deselect them with ``uncollect_if`` and ``on_galaxy``. +""" + +import pytest + +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params, torus_xy_device_params + +GALAXY_SHAPES = ((8, 4), (4, 8)) + + +def galaxy_meshes(**device_overrides) -> list: + """``mesh_device, device_params`` params for 8x4 TorusXY and 4x8 Fabric2D; fresh dicts per call.""" + return [ + pytest.param( + (8, 4), + torus_xy_device_params(**device_overrides), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(8, 4), topology="mesh-8x4"), + id="torus-xy-8x4", + ), + pytest.param( + (4, 8), + fabric2d_device_params(**device_overrides), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(4, 8), topology="mesh-4x8"), + id="fabric2d-mesh-4x8", + ), + ] + + +def on_galaxy(mesh_device) -> bool: + """Whether a ``mesh_device`` parameter (collection-time shape tuple) is a Galaxy layout.""" + return tuple(mesh_device) in GALAXY_SHAPES diff --git a/models/demos/deepseek_v3_d_p/tests/v41/lock_phases.py b/models/demos/deepseek_v3_d_p/tests/v41/lock_phases.py new file mode 100644 index 000000000000..02df18c72604 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/lock_phases.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Lock-phase breakdown of one device-test log from its timing events (models/common/timing_events.py). + +Usage:: + + python models/demos/deepseek_v3_d_p/tests/v41/lock_phases.py [] + +Prints, per test (device.open .. device.close): time from device.open to the first compute phase, seconds per +phase kind (reference, oracle, weights, compute), and cache hit/miss counts per cache; with the safe runner's +JSON result, also the runner's wall time (it includes waiting for the device lock; the lock is then held for the +whole pytest process, so runner time minus the queue wait bounds the lock-held time). +""" + +import json +import sys +from collections import Counter, defaultdict +from datetime import datetime + +PREFIX = "TT_EVENT " + + +def events(log_path: str) -> list[dict]: + out = [] + with open(log_path, errors="ignore") as log: + for line in log: + at = line.find(PREFIX) + if at >= 0: + try: + out.append(json.loads(line[at + len(PREFIX) :])) + except json.JSONDecodeError: + continue + return sorted(out, key=lambda e: e["ts"]) # pytest prints captured output per test, not chronologically + + +def _t(event: dict) -> datetime: + return datetime.fromisoformat(event["ts"]) + + +def summarize(evts: list[dict]) -> list[dict]: + """One record per device.open .. device.close span (events outside a span belong to no test).""" + runs, current = [], None + for e in evts: + if e["event"] == "device.open": + current = {"test": e.get("test"), "open": _t(e), "phases": defaultdict(float), "cache": Counter()} + current["first_compute"] = None + runs.append(current) + elif current is None: + continue + elif e["event"] == "phase.begin" and e.get("phase") == "compute" and current["first_compute"] is None: + current["first_compute"] = _t(e) + elif e["event"] == "phase.end": + current["phases"][e["phase"]] += e.get("seconds", 0.0) + elif e["event"] in ("cache.hit", "cache.miss"): + current["cache"][(e["cache"], e["event"].split(".")[1])] += 1 + elif e["event"] == "device.close": + current["close"] = _t(e) + current = None + return runs + + +def main(log_path: str, runner_json: str | None = None) -> None: + for run in summarize(events(log_path)): + to_compute = (run["first_compute"] - run["open"]).total_seconds() if run["first_compute"] else None + held = (run["close"] - run["open"]).total_seconds() if "close" in run else None + print(run["test"]) + print(f" device.open -> first compute: {to_compute if to_compute is None else f'{to_compute:.1f}s'}") + print(f" device.open -> device.close: {held if held is None else f'{held:.1f}s'}") + for kind in ("reference", "oracle", "weights", "compute"): + print(f" {kind:9s}: {run['phases'].get(kind, 0.0):8.1f}s") + for (cache, kind), n in sorted(run["cache"].items()): + print(f" cache {cache} {kind}: {n}") + if runner_json: + run = json.load(open(runner_json))["command_run"] + print( + f"runner wall time (incl. lock wait): {run['duration_seconds']:.1f}s, {run['started_at']} .. {run['ended_at']}" + ) + + +if __name__ == "__main__": + main(*sys.argv[1:3]) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/moe_reference.py b/models/demos/deepseek_v3_d_p/tests/v41/moe_reference.py new file mode 100644 index 000000000000..f38f68f1b152 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/moe_reference.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""CPU reference for the DeepSeek-V4.1 MoE op test (``test_moe_v41``); torch only, no ttnn. + +The MoE input is the §6 oracle's ``ffn_in`` of V4.1 layer 2 for one 5120-token chunk (random prompt), +synthetic (seeded) or checkpoint weights. The observable parts of ``MoE.forward`` (gate logits, routing, +routed sum, shared expert, output) are recomputed with the vendored modules and disk-cached next to the +oracle result; the CPU run takes tens of minutes, so warm the cache outside the device lock with +``python -m models.demos.deepseek_v3_d_p.tests.v41.moe_reference {synthetic,real}``. +""" + +import hashlib +import sys + +import pytest +import torch + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as O +from models.demos.deepseek_v3_d_p.tt.v41 import weights as W + +LAYER = 2 +SEQ = 5120 # one prefill chunk: 2560 tokens/chip on 2x4, 1280 on 4x2 +REFERENCE_VERSION = 1 # bump when reference_parts changes what it records + + +def spec(source: str) -> O.OracleSpec: + if source == "real" and not (O.HF_SNAPSHOT / W.INDEX_FILE).is_file(): + pytest.skip("V4.1 checkpoint shards not downloaded") + return O.real_spec((LAYER,), SEQ, checkpoint=O.HF_SNAPSHOT if source == "real" else None) + + +@torch.no_grad() +def reference_parts(moe: v41.MoE, x: torch.Tensor) -> dict: + """``MoE.forward`` (text tokens) split into its observable parts: same ops, same order.""" + logits = v41.linear(x.float(), moe.gate.weight.float()) + weights, indices = moe.gate(x) + routed = torch.zeros_like(x, dtype=torch.float32) + for e in range(moe.n_routed_experts): + token, slot = torch.where(indices == e) + if len(token): + routed[token] += moe.experts[e](x[token], weights[token, slot, None]) + shared = moe.shared_experts(x) + final = (routed + shared).type_as(x) + return { + "logits": logits, + "weights": weights, + "indices": indices, + "routed": routed, + "shared": shared, + "final": final, + } + + +def reference(source: str) -> tuple[dict, v41.Transformer | None]: + """MoE input ``x`` and reference parts for ``source`` (cached), plus the reference model if it had to be + built (synthetic device weights are read from it).""" + s = spec(source) + tokens = O.random_tokens(s) + key = hashlib.sha256(f"{O.cache_path(s, tokens).name}:{REFERENCE_VERSION}".encode()).hexdigest()[:20] + path = O.CACHE_DIR / f"moe-parts-{key}.pt" + if path.is_file(): + return torch.load(path), None + model = O.build_reference(s) + result = O.oracle(s, tokens, model) + x = result["blocks"][LAYER]["ffn_in"] + with v41.set_dtype(torch.bfloat16): + parts = reference_parts(model.layers[0].ffn, x) + assert torch.equal(parts["final"], result["blocks"][LAYER]["ffn_out"]), "MoE split differs from MoE.forward" + parts["x"] = x + tmp = path.with_suffix(".tmp") + torch.save(parts, tmp) + tmp.replace(path) + return parts, model + + +def device_weights(source: str, model: v41.Transformer | None) -> dict: + """``TtV41Moe`` weights: the checkpoint layer via the F0 loader, or the synthetic reference's, dequantized + (exact) with the F0 functions.""" + if source == "real": + return W.load_layer(W.resolve_checkpoint(), LAYER) + if model is None: + model = O.build_reference(spec(source)) + moe = model.layers[0].ffn + + def expert(e: v41.Expert, dequant) -> dict: + deq = lambda lin: dequant(lin.weight.data, lin.scale.data) + return {"gate_proj": deq(e.w1), "up_proj": deq(e.w3), "down_proj": deq(e.w2)} + + return { + "gate_weights": { + "weight": moe.gate.weight.data.to(torch.bfloat16), + "e_score_correction_bias": moe.gate.bias.data.float(), + }, + "routed_expert_weights": [expert(e, W.dequant_mxfp4) for e in moe.experts], + "shared_expert_weights": expert(moe.shared_experts, W.dequant_fp8_block), + } + + +if __name__ == "__main__": + torch.set_num_threads(16) + for src in sys.argv[1:]: + parts, _ = reference(src) + print(src, {k: tuple(v.shape) for k, v in parts.items()}, flush=True) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/prepare_caches.py b/models/demos/deepseek_v3_d_p/tests/v41/prepare_caches.py new file mode 100644 index 000000000000..88b9c95a5e41 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/prepare_caches.py @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""CPU-only cache prepare step for V4.1 device tests (bead 8y7.18.3): fills the oracle and device-weight caches +outside the device lock, so the device run only reads them. + +Usage (plain python, NOT through run_safe_pytest.sh; takes no lock and opens no hardware):: + + python models/demos/deepseek_v3_d_p/tests/v41/prepare_caches.py [ ...] + +with the same test ids as the safe runner, e.g. +``models/demos/deepseek_v3_d_p/tests/v41/test_block_v41.py::test_v41_blocks_on_device_state[blackhole-fabric2d-mesh-2x4-small-single_chunk-stack-full-bf16]``, +``.../test_transformer_v41.py::test_v41_transformer_small[blackhole-fabric2d-mesh-2x4-one_chunk-sharing]``, +``.../test_transformer_v41.py::test_v41_transformer_production[blackhole-fabric2d-mesh-2x4-real-one_chunk]``. + +It opens a MOCK mesh (TT_METAL_MOCK_CLUSTER_DESC_PATH -> the LoudBox 8x P150 descriptor: no PCIe access, no +command queues or firmware) and runs each test's own setup function (setup_blocks / setup_small / +setup_production, + reference_data for the transformer gate), so the caches are written by the canonical code: +mock-written weight-cache files are byte-identical to device-written ones. Idempotent: a second run is all hits. +""" + +import os +import re +import sys +import time +from pathlib import Path + +_REPO = Path(__file__).resolve().parents[5] +MOCK_CLUSTER = _REPO / "tt_metal/third_party/umd/tests/cluster_descriptor_examples/blackhole_8xP150.yaml" +os.environ["TT_METAL_MOCK_CLUSTER_DESC_PATH"] = str(MOCK_CLUSTER) # before ttnn is imported: never real hardware + +import pytest # noqa: E402 + +import ttnn # noqa: E402 + +_ID = re.compile(r"(?P[^:]+)::(?P\w+)\[blackhole-fabric2d-mesh-(?P\d)x(?P\d)-(?P[^\]]+)\]") +_CHUNKS = {"single_chunk": 1, "two_chunks": 2, "one_chunk": 1} + + +def _prepare(mesh, test: str, params: list[str]) -> str: + if test == "test_v41_blocks_on_device_state": + from models.demos.deepseek_v3_d_p.tests.v41.test_block_v41 import setup_blocks + + weights, chunks, schedule, prompt, kv_format = params + setup = setup_blocks(mesh, weights, _CHUNKS[chunks], schedule, prompt, kv_format) + return f"reference built: {setup.reference.built}" + from models.demos.deepseek_v3_d_p.tests.v41 import test_transformer_v41 as T + + if test == "test_v41_transformer_small": + case, schedule = params + model, spec, tokens, reference, _ = T.setup_small(mesh, case, schedule) + elif test == "test_v41_transformer_production": + weights, chunks = params + setup = T.setup_production(mesh, weights, chunks) + if setup is None: + return "skipped: checkpoint not downloaded" + model, spec, tokens, reference = setup + if chunks in T.KV_FORMAT_CASES: + T.kv_format_reference(spec, tokens, reference) + else: + T.reference_data(spec, tokens, reference, T.last_chunk_rows(chunks)) + return f"reference built: {reference.built}" + else: + raise ValueError(f"no prepare step for {test}") + T.reference_data(spec, tokens, reference) + built = getattr(reference, "built", True) + return f"reference built: {built}" + + +def main(ids: list[str]) -> None: + parsed = [] + for test_id in ids: + m = _ID.fullmatch(test_id) + if m is None: + raise SystemExit(f"cannot parse test id {test_id!r}") + parsed.append(m) + ttnn.set_fabric_config(ttnn.FabricConfig.FABRIC_2D) + meshes = {} + try: + for m in parsed: + shape = (int(m["sp"]), int(m["tp"])) + if shape not in meshes: + meshes[shape] = ttnn.open_mesh_device(ttnn.MeshShape(*shape)) + start = time.perf_counter() + try: + note = _prepare(meshes[shape], m["test"], m["params"].split("-")) + except pytest.skip.Exception as skip: + note = f"skipped: {skip}" + print(f"prepare {m['test']}[{m['params']}]: {note}, {time.perf_counter() - start:.1f}s", flush=True) + finally: + for mesh in meshes.values(): + ttnn.close_mesh_device(mesh) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/reference_weights.py b/models/demos/deepseek_v3_d_p/tests/v41/reference_weights.py new file mode 100644 index 000000000000..093469adf263 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/reference_weights.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Device-layout weight dicts built from vendored-reference modules (a test helper, not an oracle). + +``device_weights`` turns a reference ``Block`` (e.g. from ``oracle.build_reference``) into the ``TtV41Block`` +``weights`` dict; ``dequant`` gives a reference ``Linear``'s weight as bf16 ``[out, in]``. +""" + +from __future__ import annotations + +import torch + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.kernel_cpu import unpack_fp4 + + +def dequant(linear) -> torch.Tensor: + """A reference Linear's weight as bf16 [out, in] (FP8 32x32 / FP4 per-32 dequantization is exact).""" + w = linear.weight + if w.dtype == torch.float8_e4m3fn: + s = linear.scale.float() + full = s.repeat_interleave(32, 0)[: w.shape[0]].repeat_interleave(32, 1)[:, : w.shape[1]] + return (w.float() * full).to(torch.bfloat16) + if w.dtype == torch.float4_e2m1fn_x2: + return (unpack_fp4(w) * linear.scale.float().repeat_interleave(32, 1)).to(torch.bfloat16) + return w.detach().to(torch.bfloat16) + + +MOE_KEYS = ("gate_weights", "routed_expert_weights", "shared_expert_weights") + + +@torch.no_grad() +def device_weights(model: v41.Transformer, layer: int, include_moe: bool = True) -> dict: + """The TtV41Block ``weights`` dict of the reference backbone layer at position ``layer``. With + ``include_moe=False`` the MoE entries (``MOE_KEYS``) are omitted and not computed (the routed experts' FP4 + dequantization dominates the cost), for callers whose device MoE tensors are cached.""" + blk = model.layers[layer] + attn, ffn = blk.attn, blk.ffn + extra = {} + if attn.compressor is not None: + c = attn.compressor + extra["compressor"] = {"wkv": c.wkv.weight.detach(), "norm": c.norm.weight.detach()} | ( + {"wgate": c.wgate.weight.detach()} if c.compress_ratio > 1 else {} + ) + if attn.indexer is not None: + ind = attn.indexer + extra["indexer"] = {"wq_b": dequant(ind.wq_b), "weights_proj": ind.weights_proj.weight.detach()} | ( + {"wk": ind.wk.weight.detach(), "k_norm": ind.k_norm.weight.detach()} if ind.owns_k else {} + ) + weights = extra | { + "attn": { + "wq_a": dequant(attn.wq_a), + "q_norm": attn.q_norm.weight.detach(), + "wq_b": dequant(attn.wq_b), + "wkv": dequant(attn.wkv), + "kv_norm": attn.kv_norm.weight.detach(), + "wo_a": dequant(attn.wo_a), + "wo_b": dequant(attn.wo_b), + "attn_sink": attn.attn_sink.detach(), + }, + "attn_norm": blk.attn_norm.weight.detach(), + "ffn_norm": blk.ffn_norm.weight.detach(), + "hc_attn": (blk.hc_attn_fn.detach(), blk.hc_attn_base.detach(), blk.hc_attn_scale.detach()), + "hc_ffn": (blk.hc_ffn_fn.detach(), blk.hc_ffn_base.detach(), blk.hc_ffn_scale.detach()), + } + if not include_moe: + return weights + se = ffn.shared_experts + return weights | { + "gate_weights": { + "weight": ffn.gate.weight.detach().to(torch.bfloat16), + "e_score_correction_bias": ffn.gate.bias.detach().float(), + }, + "routed_expert_weights": [ + {"gate_proj": dequant(e.w1), "up_proj": dequant(e.w3), "down_proj": dequant(e.w2)} for e in ffn.experts + ], + "shared_expert_weights": { + "gate_proj": dequant(se.w1), + "up_proj": dequant(se.w3), + "down_proj": dequant(se.w2), + }, + } diff --git a/models/demos/deepseek_v3_d_p/tests/v41/small_config.py b/models/demos/deepseek_v3_d_p/tests/v41/small_config.py new file mode 100644 index 000000000000..f25ec32f237d --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/small_config.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""A small DeepSeek-V4.1 configuration for fast device iteration. + +Same schedule rules, block types and code paths as the released model, with every dimension shrunk while +keeping the device constraints (hidden and q_lora split over TP=4 in whole tiles, >= 32 heads after the +head->sequence reshard, 32-wide QDQ groups, experts at the smallest in-tree-tested count, top-k a multiple of 16). +Oracles at these dims take seconds on CPU; real dims stay the acceptance runs. +""" + +from dataclasses import replace + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig + + +class SmallV41Config(DeepSeekV41FlashConfig): + EMB_SIZE = 1024 + FABRIC_PAYLOAD_SIZE = EMB_SIZE + MOE_INTERMEDIATE_SIZE = 256 + NUM_ROUTED_EXPERTS = 128 # smallest count the in-tree MoE dispatch is exercised with (32 hung in routing setup) + NUM_ATTENTION_HEADS = 32 + HEAD_DIM = 128 + Q_LORA_RANK = 256 + O_LORA_RANK = 128 + INDEX_N_HEADS = 32 + INDEX_HEAD_DIM = 128 + INDEX_TOPK = 128 # selected / visible rows ~ 0.5 at S=512, ratio 2 (real: 512 of 1024 at S=2048) + CANDIDATE_TOPK_BLOCKS = 32 # 32 x 8 = 256 >= INDEX_TOPK rows, half of the 64 visible blocks + + +def small_spec(layers: tuple[int, ...], seq_len: int, *, seed: int = 0, dspark: bool = False): + """An oracle spec for V4.1 layers ``layers`` (their real roles, plus the DSpark layers with ``dspark``) at + SmallV41Config dims, synthetic weights.""" + c = SmallV41Config + spec = orc.real_spec(layers, seq_len, dspark=dspark, candidate_topk_blocks=c.CANDIDATE_TOPK_BLOCKS, seed=seed) + args = replace( + spec.args, + dim=c.EMB_SIZE, + moe_inter_dim=c.MOE_INTERMEDIATE_SIZE, + n_routed_experts=c.NUM_ROUTED_EXPERTS, + n_heads=c.NUM_ATTENTION_HEADS, + head_dim=c.HEAD_DIM, + q_lora_rank=c.Q_LORA_RANK, + o_lora_rank=c.O_LORA_RANK, + index_n_heads=c.INDEX_N_HEADS, + index_head_dim=c.INDEX_HEAD_DIM, + index_topk=c.INDEX_TOPK, + ) + return replace(spec, args=args, model_args=replace(spec.model_args, dim=c.EMB_SIZE)) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/spec_sample.py b/models/demos/deepseek_v3_d_p/tests/v41/spec_sample.py new file mode 100644 index 000000000000..0e4e3d8a315c --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/spec_sample.py @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The sampling-correct speculative round: exactness contract as a CPU reference. + +Stdlib only. The target model is abstract: a callable from +the committed prefix to the logits of the next token. The served sampler is modelled as the device tail +(``tt/v41/batch_decode.py:DeviceSampler`` and ``ttnn.sampling``'s kernels, +``ttnn/cpp/ttnn/operations/reduction/sampling/device/kernels/{compute/sampling.cpp,dataflow/writer_interleaved.cpp}``): + +- the candidates are the row's top-128 logits (four TP chips' top-32 each), of which the kernel keeps the + **top-32** (``nearest32_K = 32``) and masks all but the row's ``k`` (``k`` in 1..32: the generator clamps + the request's ``top_k`` to 32 and maps "no top_k" to 32, ``tt/v41/spec_verify.py:_host_inputs``); +- the kept logits are scaled by ``1 / T`` and soft-maxed (``mul_block_bcast_scalar_inplace``, ``reduce_c``, + ``sub_exp_block_bcast_cols_inplace``); +- the nucleus is the shortest sorted prefix whose cumulative probability **exceeds** ``p`` + (``writer_interleaved.cpp``: ``if (cum_prob_f > p_f) { top_p_cutoff = i + 1; break; }``; ``p = 0`` keeps the + top candidate alone: the plugin's argmax convention for ``top_k == 1``, ``input_batch.py:343-353``); +- one uniform ``u`` in ``[0, 1)`` per row per call (``rand_tile``; the cores' generators run across replays), + the token is the first of the nucleus whose renormalised cumulative probability exceeds ``u``. + +``served_probs`` is that distribution, ``served_draw`` one draw from it. Temperature 0 is the argmax +(``k = 1, p = 0``), as the generator sends it. + +**The round.** Position convention: the block ``[x, d_1 .. d_K]`` is verified at +positions ``p .. p + K``; row ``j`` (the row of input ``j``) carries the target's distribution ``P_j`` for +position ``p + j + 1`` given the committed prefix plus ``x, d_1 .. d_j``. The drafter is **greedy**: its +proposal ``q_j`` is a point mass at ``d_{j+1}``. + +Two rules, both exact, both implemented here: + +- ``round_reject`` (standard speculative sampling, Leviathan et al. 2023 / Chen et al. 2023, with ``q`` a +point + mass): at row ``j``, accept ``d_{j+1}`` with probability ``min(1, P_j(d) / q_j(d)) = P_j(d)``; on rejection + draw from the residual ``max(0, P_j - q_j)`` normalised, which for the point mass is ``P_j`` with ``d`` + removed and renormalised by ``1 - P_j(d)``; after ``K`` accepts draw the bonus from ``P_K``. Two uniforms a + row at most (the accept test and the residual draw). +- ``round_match`` (target-sample / match): at row ``j`` draw + ``x_j ~ P_j``; if ``j < K`` and ``x_j == d_{j+1}`` accept and continue, else commit ``x_j`` and stop. One + uniform a row. This is the device step as it stands (``ttnn.sampling`` on every verify row with the + request's ``k, p, T``) followed by the greedy ``accept`` rule (``spec_verify.py:accept``: the longest prefix + with ``block[a + 1] == sampled[a]``), so the served path needs **no new kernel**. + +**Proof that both commit the target's distribution.** Fix the committed prefix and the drafts. Under +``round_reject`` the token committed at row ``j`` (given rows ``0 .. j - 1`` accepted, i.e. given the prefix +``x, d_1 .. d_j``) is ``d`` with probability ``P_j(d)`` and ``y != d`` with probability +``(1 - P_j(d)) * P_j(y) / (1 - P_j(d)) = P_j(y)``: the committed token is a draw from ``P_j``. Under +``round_match`` it is ``x_j ~ P_j`` by construction. In both, "accept" means "the committed token equals the +draft", which happens with probability ``P_j(d_{j+1})`` and leaves the prefix for row ``j + 1`` equal to the +prefix the next sequential draw would condition on; the round stops at the first committed token that is not +the next draft, and the next round's rows condition on exactly the committed tokens. By induction over the +stream, the ``n``-th committed token is distributed as ``P(. | the first n - 1 committed tokens)``: the +sequential sampler's law. Nothing depends on how the drafts were chosen (the proof holds for any +deterministic ``d``; a draft outside the nucleus is simply rejected with probability one). The two rules +differ only in how they spend uniforms; their committed distributions are identical. At ``T = 0`` every +``P_j`` is a point mass at the argmax and both rules reduce to the greedy longest-prefix match. + +What is **not** exact and why it is the served contract anyway: ``P_j`` is the served top-32 / top-p / ``T`` +tail, not the full-vocabulary softmax. The plain served step draws from the same tail, so "exact" here means +"the same distribution as plain serving on this stack", which is what +the served sampler and the benchmark honour (temperature 1, no ``top_k``, ``top_p`` 1: ``k = 32``, every +nucleus the 32 candidates). + +""" + +from __future__ import annotations + +import math +import random +from collections.abc import Callable, Sequence + +Logits = Sequence[float] +TargetFn = Callable[[Sequence[int]], Logits] # the committed prefix -> the next token's logits +DrafterFn = Callable[[Sequence[int], int], list[int]] # the prefix (ending in x) and K -> the K drafts + +CANDIDATES = 32 # the kernel's top-32 of the gathered top-128 +TOPK_MAX = 32 # the generator clamps the request's top_k to it; 0 / none -> 32 + + +def _sorted_candidates(logits: Logits) -> list[tuple[float, int]]: + """The row's candidates in the kernel's order: descending value, ties by the lowest index (stable top-k).""" + order = sorted(range(len(logits)), key=lambda i: (-float(logits[i]), i)) + return [(float(logits[i]), i) for i in order[:CANDIDATES]] + + +def served_probs(logits: Logits, k: int, p: float, temperature: float) -> list[tuple[int, float]]: + """The served sampler's distribution over the row: ``[(token, probability)]`` in the kernel's order (the + nucleus; probabilities sum to 1). ``temperature <= 0`` is the argmax (``k = 1, p = 0``).""" + if temperature <= 0: + k, p, temperature = 1, 0.0, 1.0 + k = TOPK_MAX if k <= 0 else max(1, min(int(k), TOPK_MAX)) + cand = _sorted_candidates(logits)[:k] + scaled = [v / temperature for v, _ in cand] + m = max(scaled) + w = [math.exp(s - m) for s in scaled] + z = sum(w) + probs = [x / z for x in w] + cum, nucleus = 0.0, len(probs) + for i, q in enumerate(probs): + cum += q + if cum > p: + nucleus = i + 1 + break + kept = probs[:nucleus] + z = sum(kept) + return [(cand[i][1], kept[i] / z) for i in range(nucleus)] + + +def draw(dist: Sequence[tuple[int, float]], u: float) -> int: + """The kernel's multinomial: the first token whose cumulative probability exceeds ``u`` (the last one if + rounding leaves ``u`` above the sum).""" + cum = 0.0 + for token, q in dist: + cum += q + if cum > u: + return token + return dist[-1][0] + + +def served_draw(logits: Logits, k: int, p: float, temperature: float, rng: random.Random) -> int: + return draw(served_probs(logits, k, p, temperature), rng.random()) + + +def prob_of(dist: Sequence[tuple[int, float]], token: int) -> float: + for t, q in dist: + if t == token: + return q + return 0.0 + + +def residual(dist: Sequence[tuple[int, float]], token: int) -> list[tuple[int, float]]: + """``max(0, P - q)`` normalised for a point mass ``q`` at ``token``: ``P`` without ``token``.""" + rest = [(t, q) for t, q in dist if t != token] + z = sum(q for _, q in rest) + assert z > 0, "the residual is empty: the target put all its mass on the draft" + return [(t, q / z) for t, q in rest] + + +class Params: + """A request's sampling parameters as the generator hands them to the step.""" + + def __init__(self, k: int = 0, p: float = 1.0, temperature: float = 1.0): + self.k, self.p, self.temperature = k, p, temperature + + def probs(self, logits: Logits) -> list[tuple[int, float]]: + return served_probs(logits, self.k, self.p, self.temperature) + + +def round_match( + target: TargetFn, prefix: Sequence[int], block: Sequence[int], params: Params, rng: random.Random +) -> tuple[int, list[int]]: + """The target-sample / match rule over the block ``[x, d_1 .. d_K]`` verified after ``prefix``: the + accepted + count ``a`` and the emitted tokens (``a`` drafts and the committed token of row ``a``). One uniform a row, + rows after the first mismatch never drawn (the device draws them; the rule ignores them).""" + history = list(prefix) + emitted: list[int] = [] + a = 0 + for j, token in enumerate(block): + history.append(token) + x = draw(params.probs(target(history)), rng.random()) + if j < len(block) - 1 and x == block[j + 1]: + a += 1 + emitted.append(x) + continue + emitted.append(x) + break + return a, emitted + + +def round_reject( + target: TargetFn, prefix: Sequence[int], block: Sequence[int], params: Params, rng: random.Random +) -> tuple[int, list[int]]: + """Standard speculative sampling with a point-mass proposal: accept ``d_{j+1}`` with probability + ``P_j(d_{j+1})`` (the uniform below it), else draw from the residual; the bonus from ``P_K``.""" + history = list(prefix) + emitted: list[int] = [] + a = 0 + for j, token in enumerate(block): + history.append(token) + dist = params.probs(target(history)) + if j < len(block) - 1: + d = block[j + 1] + if rng.random() < prob_of(dist, d): + a += 1 + emitted.append(d) + continue + emitted.append(draw(residual(dist, d), rng.random())) + break + emitted.append(draw(dist, rng.random())) + return a, emitted + + +def round_wrong( + target: TargetFn, prefix: Sequence[int], block: Sequence[int], params: Params, rng: random.Random +) -> tuple[int, list[int]]: + """The negative control: greedy acceptance (the draft equals the row's argmax) with a sampled bonus. Not + exact: it commits the draft whenever it is the mode, so the mode is over-represented.""" + history = list(prefix) + emitted: list[int] = [] + a = 0 + for j, token in enumerate(block): + history.append(token) + dist = params.probs(target(history)) + mode = max(dist, key=lambda tq: tq[1])[0] + if j < len(block) - 1 and block[j + 1] == mode: + a += 1 + emitted.append(mode) + continue + emitted.append(draw(dist, rng.random())) + break + return a, emitted + + +RoundFn = Callable[[TargetFn, Sequence[int], Sequence[int], Params, random.Random], tuple[int, list[int]]] + + +def plain_stream(target: TargetFn, prompt: Sequence[int], n: int, params: Params, rng: random.Random) -> list[int]: + """``n`` tokens after ``prompt`` by the served sampler, one a step: the law every round must match.""" + history = list(prompt) + out: list[int] = [] + for _ in range(n): + x = draw(params.probs(target(history)), rng.random()) + history.append(x) + out.append(x) + return out + + +def spec_stream( + target: TargetFn, + drafter: DrafterFn, + prompt: Sequence[int], + n: int, + k: int, + params: Params, + rng: random.Random, + round_fn: RoundFn = round_match, +) -> tuple[list[int], list[int]]: + """``n`` tokens after ``prompt`` by speculative rounds of ``k`` drafts (the first token of the stream is a + plain draw, as the served request's first step is a plain row): the tokens and the accepted count of every + round.""" + history = list(prompt) + out: list[int] = [] + x = draw(params.probs(target(history)), rng.random()) + out.append(x) + accepted: list[int] = [] + while len(out) < n: + drafts = drafter(history + [x], k) + a, emitted = round_fn(target, history, [x] + drafts, params, rng) + accepted.append(a) + history += [x] + emitted[:-1] + x = emitted[-1] + out += emitted + return out[:n], accepted diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_block_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_block_v41.py new file mode 100644 index 000000000000..9323689363c2 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_block_v41.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 blocks on device state (beads F3, F7): real-dims schedule 2 -> 3 -> 20 -> 21 -> 24. + +Each block runs on device with the oracle's inputs (teacher-forced streams and pre_mix); everything else is +device-produced and device-held: window KV, compressed KV and index keys written by the KV sources, top-k +published by the index sources and reused by consumers, candidates of layer 20 constraining layer 24. +Checks per layer: block output and next pre_mix vs the oracle, compressed-KV and index-K rows vs the oracle's +shared state, the window-KV carry vs the oracle's window KV, determinism (a second pass over a fresh state is +bit-identical); selection recall is reported. +Bars (G1): block >= 0.99 synthetic / >= 0.98 real; caches >= 0.998. + +``kv_format`` selects the stage of the compressed layers' KV (epic KV FORMAT): ``bf16`` stores the reference's +QDQ values, so compressed KV is compared with the oracle's FP4-QDQ rows; ``scaled_fp8`` encodes the unrounded +KV, so its compressed rows are compared with the reference's compressed KV before the FP4 QDQ (the value the +format encodes); the PCC against the FP4-QDQ rows is reported (``compressed_kv_vs_fp4``, no bar). +""" + +import os +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +from loguru import logger + +import ttnn +from models.common import timing_events +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import kernel_cpu +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes, on_galaxy +from models.demos.deepseek_v3_d_p.tests.v41.reference_weights import MOE_KEYS, device_weights +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config, small_spec +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import ( # noqa: F401 (WEIGHT_CACHE re-export) + WEIGHT_CACHE, + begin_layer, + complete_layer, + host_weights, + weight_cache_dir, +) +from models.demos.deepseek_v3_d_p.tt.v41.block import TtV41Block +from models.demos.deepseek_v3_d_p.tt.v41.cache import WINDOW_SLOT, V41PrefillState +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint +from models.demos.deepseek_v3_d_p.utils.fast_cache_checker import init_checker +from models.demos.deepseek_v3_d_p.utils.kv_cache_utils import MlaKvCacheFormat +from tests.ttnn.utils_for_testing import comp_pcc + +# every sharing role (2->3, 20->21->24) and the sliding-window-only type (layer 0; layer 1 needs Engram, F5) +SCHEDULES = {"stack": (2, 3, 20, 21, 24), "swa": (0,)} +SEQ = 2048 +BLOCK_PCC = {"small": 0.99, "synthetic": 0.99, "real": 0.98} +# User decision 2026-09-29 (bars by data source): synthetic production-shape index-source blocks are hypersensitive +# to the intrinsic top-k disagreement of FP4-quantized selection (reference self-recall 0.993 at bf16-level input +# perturbation; synthetic compressed rows uncorrelated), so they are gated at the real-weight bar. +SYNTHETIC_INDEX_SOURCE_PCC = 0.98 +SMALL_SEQ = 512 +CACHE_PCC = 0.998 + + +def _pack(x, tp): + s, n, d = x.shape + return x.reshape(s, n, tp, d // tp).permute(0, 2, 1, 3).reshape(1, 1, s, n * d) + + +def _unpack(t, n, tp): + s, width = t.shape[-2], t.shape[-1] + return t.reshape(s, tp, n, width // n // tp).permute(0, 2, 1, 3).reshape(s, n, -1) + + +def _pcc(a, b): + return comp_pcc(a.float(), b.float(), 0.0)[1] + + +KV_FORMATS = {"bf16": MlaKvCacheFormat.BF16_RM, "scaled_fp8": MlaKvCacheFormat.SCALED_FP8} + + +@torch.no_grad() +def _unrounded_compressed_kv(model, spec, layer: int, attn_in: torch.Tensor) -> torch.Tensor: + """The reference compressed KV of KV source ``layer`` with RoPE and before its FP4 QDQ (``Attention._compress_kv`` + at start_pos 0), from the recorded attention input: what the SCALED_FP8 cache encodes.""" + attn = model.layers[spec.layer_ids.index(layer)].attn + seq, ratio = attn_in.shape[0], attn.compress_ratio + with v41.set_dtype(torch.bfloat16): + latent = attn.compressor(attn_in[None], 0) + if latent is None: # shorter than one group + return attn_in.new_zeros(0, attn.head_dim) + v41.apply_rotary_emb(latent[..., -attn.rope_head_dim :], attn.freqs_cis[: seq - seq % ratio : ratio]) + return latent[0] + + +def unrounded_kv(spec, tokens, reference, result) -> dict[int, torch.Tensor]: + """KV source layer -> its unrounded compressed KV (what the SCALED_FP8 cache encodes), disk-cached next to the + oracle ``result`` of ``tokens``; ``reference`` (a LazyReference) is built only on a miss.""" + unrounded = {} + for l in (l for l in spec.layer_ids if l in C.KV_SOURCE_LAYERS): + base = orc.cache_path(spec, tokens) + unrounded[l] = host_weights( + base.parent, + f"{base.stem}-unrounded-kv{l}", + lambda l=l: _unrounded_compressed_kv(reference(), spec, l, result["blocks"][l]["attn_in"]), + ) + # the derivation is the reference's: its FP4 QDQ reproduces the oracle's compressed rows bit for bit + fp4 = kernel_cpu.fp4_act_quant(unrounded[l].clone(), 16, True, scale_dtype=torch.float8_e4m3fn) + assert torch.equal(fp4, result["shared"][l]["compress_kv"]), l + return unrounded + + +def setup_blocks(mesh_device, weights, chunks, schedule, prompt, kv_format): + """Everything before the device forward: oracle result, reference-derived expectations, device weights (MoE + tensors from / into the weight cache) and the blocks. Shared by the test and the CPU-only cache prepare step + (tests/v41/prepare_caches.py), which runs it on a mock mesh so the device run only reads caches.""" + LAYERS = SCHEDULES[schedule] + ckpt = resolve_checkpoint() if weights == "real" else None + if weights == "real" and ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + if weights == "small": # same code path at small dims: seconds of CPU oracle, minutes of device time + seq = SMALL_SEQ + spec, cfg = small_spec(LAYERS, seq), SmallV41Config + else: + seq = SEQ + spec = orc.real_spec(LAYERS, seq, candidate_topk_blocks=96, checkpoint=ckpt.root if ckpt else None) + cfg = type("V41TestConfig", (C,), {"CANDIDATE_TOPK_BLOCKS": 96}) # match the oracle's candidate count + # prompt lengths: full chunks, a padded last chunk (graph.md rule 8), and a prompt shorter than a ratio-2 group + # pair plus window edge (empty or single compressed row: top-k empty) + valid = {"full": seq, "padded": seq - 12, "tiny": 3}[prompt] + if prompt != "full" and weights != "small": + pytest.skip("prompt-length edge cases run at small dims") + tokens = orc.random_tokens(spec, valid) + # built only on an oracle or weight-cache miss (warm runs construct no reference model) + reference = orc.LazyReference(spec) + result = orc.oracle(spec, tokens, model=reference) + fmt = KV_FORMATS[kv_format] + unrounded = unrounded_kv(spec, tokens, reference, result) if fmt == MlaKvCacheFormat.SCALED_FP8 else {} + + # MoE device tensors are cached on disk (tests/v41/weight_cache.py key): the first build converts 1152 expert + # matrices per layer on the host (minutes); later builds load them. A marker records a completed layer. + cache_root = weight_cache_dir(spec, mesh_device.shape) + cache_root.mkdir(parents=True, exist_ok=True) + init_checker(cache_root) + blocks = {} + with timing_events.phase("weights", layers=list(LAYERS)): + for i, layer in enumerate(LAYERS): + cached = begin_layer(cache_root, layer) + if ckpt is not None: + w = load_layer_dense(ckpt, layer) if cached else load_layer(ckpt, layer) + elif cached: + w = host_weights( + cache_root, f"layer_{layer}.dense", lambda i=i: device_weights(reference(), i, include_moe=False) + ) + else: + w = device_weights(reference(), i) + host_weights( + cache_root, f"layer_{layer}.dense", lambda w=w: {k: v for k, v in w.items() if k not in MOE_KEYS} + ) + blocks[layer] = TtV41Block(mesh_device, cfg, layer, w, seq // chunks, weight_cache_path=cache_root) + complete_layer(cache_root, layer) + del w + logger.info(f"reference model built: {reference.built}") + + return SimpleNamespace( + LAYERS=LAYERS, + seq=seq, + spec=spec, + cfg=cfg, + valid=valid, + tokens=tokens, + result=result, + fmt=fmt, + unrounded=unrounded, + blocks=blocks, + reference=reference, + ) + + +@pytest.mark.timeout(5400) +@pytest.mark.parametrize("kv_format", list(KV_FORMATS)) +@pytest.mark.parametrize("prompt", ["full", "padded", "tiny"]) +@pytest.mark.parametrize("schedule", ["stack", "swa"]) +@pytest.mark.parametrize("chunks", [1, 2], ids=["single_chunk", "two_chunks"]) +@pytest.mark.parametrize("weights", ["small", "synthetic", "real"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ), + *galaxy_meshes(), + ], + indirect=True, +) +@pytest.mark.uncollect_if(pred=lambda mesh_device, weights, **_: on_galaxy(mesh_device) and weights == "small") +def test_v41_blocks_on_device_state(mesh_device, device_params, weights, chunks, schedule, prompt, kv_format): + setup = setup_blocks(mesh_device, weights, chunks, schedule, prompt, kv_format) + LAYERS, seq, spec, cfg, valid, tokens = setup.LAYERS, setup.seq, setup.spec, setup.cfg, setup.valid, setup.tokens + result, fmt, unrounded, blocks = setup.result, setup.fmt, setup.unrounded, setup.blocks + shape, (sp, tp), n = tuple(mesh_device.shape), tuple(mesh_device.shape), cfg.HC_MULT + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + + def run(): + """All chunks through all layers in execution order; layer inputs teacher-forced per chunk, padded rows + of the last chunk zero (their outputs are not compared).""" + chunk = seq // chunks + state = V41PrefillState(mesh_device, cfg, seq, chunk, list(LAYERS), kv_format=fmt) + outs = {layer: ([], []) for layer in LAYERS} + for c in range(chunks): + start = c * chunk + length = min(chunk, valid - start) + if length <= 0: + break + for layer in LAYERS: + rec = result["blocks"][layer] + x_rows = torch.zeros(chunk, *rec["x_in"].shape[1:]) + x_rows[:length] = rec["x_in"][start : start + length].float() + pre_rows = torch.zeros(chunk, rec["pre_in"].shape[-1]) + pre_rows[:length] = rec["pre_in"][start : start + length].float() + x = ttnn.from_torch( + _pack(x_rows, tp), + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + pre = ttnn.from_torch( + pre_rows[None, None], + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, None)), + ) + x_out, pre_out = blocks[layer](x, pre, state, length) + outs[layer][0].append(_unpack(down(x_out)[0, 0], n, tp)[:length]) + outs[layer][1].append(down(pre_out)[0, 0, :length, :n]) + state.advance(length) + return state, {layer: (torch.cat(a), torch.cat(b)) for layer, (a, b) in outs.items()} + + with timing_events.phase("compute", runs=2): + state, first = run() + _, second = run() + report = {} + # the first layer's attention alone on the oracle's attention input (a KV source reads only its own rows) + head = LAYERS[0] + attention_pcc = None + if prompt == "full" and chunks == 1: + # the first layer's attention alone on the oracle's attention input (a KV source reads only its own rows) + attn_state = V41PrefillState(mesh_device, cfg, seq, seq, [head], kv_format=fmt) + attn_in = ttnn.from_torch( + result["blocks"][head]["attn_in"][None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + attention_pcc = _pcc( + result["blocks"][head]["attn_out"], down(blocks[head].attn(attn_in, attn_state, seq))[0, 0] + ) + for layer in LAYERS: + rec, (x_out, pre_out) = result["blocks"][layer], first[layer] + entry = { + "block_type": C.block_type(layer).value, + "block_out": _pcc(rec["x_out"], x_out), + "pre_mix": _pcc(rec["pre_out"], pre_out), + "deterministic": torch.equal(x_out, second[layer][0]) and torch.equal(pre_out, second[layer][1]), + } + # window KV contents: the carry holds the last WINDOW_SLOT positions' post-RoPE FP8-QDQ rows, right-aligned + tail = rec["window_kv"][max(0, valid - WINDOW_SLOT) : valid].float() + expected_carry = torch.zeros(WINDOW_SLOT, tail.shape[-1]) + expected_carry[WINDOW_SLOT - tail.shape[0] :] = tail + entry["window_kv"] = _pcc(expected_carry, state.to_host(state.window_carry[layer])) + if layer in C.KV_SOURCE_LAYERS: + pub = result["shared"][layer] + rows = pub["compress_kv"].shape[0] + w0 = state.geometry.window_rows + stored = state.to_host(state.kv[layer])[w0 : w0 + rows] + if fmt == MlaKvCacheFormat.SCALED_FP8: + entry["compressed_kv"] = _pcc(unrounded[layer], stored) + entry["compressed_kv_vs_fp4"] = _pcc(pub["compress_kv"], stored) + else: + entry["compressed_kv"] = _pcc(pub["compress_kv"], stored) + entry["index_k"] = _pcc(pub["index_k"], state.to_host(state.index_k[layer])[:rows]) + if layer == head and attention_pcc is not None: + entry["attention_alone"] = attention_pcc + report[layer] = entry + logger.info(f"layer {layer} ({weights}): {entry}") + out = os.environ.get("TT_V41_BLOCK_REPORT") + if out: + Path(out).write_text(repr(report)) + for layer, entry in report.items(): + assert entry["deterministic"], (layer, entry) + bar = BLOCK_PCC[weights] + if weights == "synthetic" and layer in C.INDEX_SOURCE_LAYERS: + bar = SYNTHETIC_INDEX_SOURCE_PCC + assert entry["block_out"] >= bar, (layer, bar, entry) + for key in ("compressed_kv", "index_k", "window_kv"): + if key in entry: + assert entry[key] >= CACHE_PCC, (layer, key, entry) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_cache_layout.py b/models/demos/deepseek_v3_d_p/tests/v41/test_cache_layout.py new file mode 100644 index 000000000000..6dbb8fa38a6f --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_cache_layout.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 prefill cache layout (bead F6.3): geometry goldens, device writes over chunks, and the device-side +zero initialization of large caches (bead F10).""" + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41.cache import ( + WINDOW_SLOT, + ZERO_BLOCK_BYTES, + V41CacheGeometry, + V41PrefillState, + replicated_zeros, +) +from models.demos.deepseek_v3_d_p.utils.kv_cache_utils import MlaKvCacheFormat + + +def test_geometry_goldens(expect_error): + g = V41CacheGeometry(C, max_seq_len=20480, chunk=5120, sp=2) + assert g.window_rows == 128 + 5120 + assert g.kv_rows(2) == 5248 + 10240 and g.kv_rows(1) == 5248 + 20480 and g.kv_rows(0) == 5248 + assert g.new_compressed_rows(5120, 5120, 2) == (2560, 2560) + assert g.new_compressed_rows(10240, 1001, 2) == (5120, 500) # odd valid length: last group incomplete + assert g.new_compressed_rows(0, 1001, 1) == (0, 1001) + assert g.kv_row_of_compressed(7) == 5248 + 7 + assert g.kv_row_of_window(5120 - 127, 5120) == 1 and g.kv_row_of_window(5120 + 3, 5120) == 131 + with expect_error(AssertionError, "multiple of 2\\*32\\*sp"): + V41CacheGeometry(C, max_seq_len=20480, chunk=5120 + 64, sp=2) + + +MESH_2X4 = [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) +] + + +@pytest.mark.parametrize("mesh_device, device_params", MESH_2X4, indirect=True) +def test_state_writes_over_chunks(mesh_device, device_params): + """Two full chunks and a padded tail through SP-sharded rows; host model of the layout as expected.""" + chunk, max_seq = 1024, 3072 + layers = [0, 2, 3, 20] # ratio 0, ratio-2 source, its consumer, ratio-1 source + state = V41PrefillState(mesh_device, C, max_seq, chunk, layers) + shape = tuple(mesh_device.shape) + gen = torch.Generator().manual_seed(3) + d, idim = C.HEAD_DIM, C.INDEX_HEAD_DIM + expect_kv = {l: torch.zeros(state.geometry.kv_rows(C.compress_ratio(l)), d) for l in (2, 20)} + expect_idx = {l: torch.zeros(state.geometry.compressed_rows(C.compress_ratio(l)), idim) for l in (2, 20)} + expect_carry = {l: torch.zeros(WINDOW_SLOT, d) for l in layers} + + def sp_sharded(t): + return ttnn.from_torch( + t[None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, None)), + ) + + def host(t): + return ttnn.to_torch(ttnn.get_device_tensors(t)[0]).float()[0, 0] + + for length in (chunk, chunk, 700): + start = state.start + for source in (2, 20): + r = C.compress_ratio(source) + rows_kv = torch.randn(chunk // r, d, generator=gen).to(torch.bfloat16) + rows_ik = torch.randn(chunk // r, idim, generator=gen).to(torch.bfloat16) + state.write_compressed(source, sp_sharded(rows_kv), sp_sharded(rows_ik), length) + first, count = state.geometry.new_compressed_rows(start, length, r) + w = state.geometry.window_rows + expect_kv[source][w + first : w + first + count] = rows_kv[:count].float() + expect_idx[source][first : first + count] = rows_ik[:count].float() + for layer in layers: + window = torch.randn(chunk, d, generator=gen).to(torch.bfloat16) + dst = state.write_window(layer, sp_sharded(window)) + region = torch.cat([expect_carry[layer], window.float()]) + assert torch.equal(host(dst)[: state.geometry.window_rows], region), f"layer {layer} window region" + state.update_window_carry(layer, length) + expect_carry[layer] = region[length : length + WINDOW_SLOT] + assert torch.equal(host(state.window_carry[layer]), expect_carry[layer]), f"layer {layer} carry" + state.advance(length) + + for source in (2, 20): + w = state.geometry.window_rows + assert torch.equal(host(state.kv[source])[w:], expect_kv[source][w:]), f"compressed KV of {source}" + assert torch.equal(host(state.index_k[source]), expect_idx[source]), f"index-K of {source}" + assert state.kv_tensor(3) is state.kv[2] and state.kv_tensor(0) is state.swa_scratch + + +def _all_chips(mesh_device, t) -> torch.Tensor: + """Raw stored values of every chip's replica, [chips, rows, width] (FP8 leaves through a mesh composer).""" + return ttnn.to_torch(t, mesh_composer=ttnn.ConcatMeshToTensor(mesh_device, dim=0))[:, 0].float() + + +@pytest.mark.parametrize("mesh_device, device_params", MESH_2X4, indirect=True) +def test_large_cache_zeroed_on_device(mesh_device, device_params): + """Caches above ZERO_BLOCK_BYTES are zeroed by device copies of one uploaded block, on every chip, over memory + that held non-zero data, for both KV storage formats; a tensor within one block is a plain upload.""" + width = 512 + step = ZERO_BLOCK_BYTES // (width * 2) + rows = 2 * step + 96 # two whole blocks and a partial one + # dirty the DRAM the zero tensors are likely to reuse + junk = ttnn.from_torch( + torch.full((1, 1, 4 * rows, width), 7.0), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + ttnn.deallocate(junk) + for dtype in (ttnn.bfloat16, MlaKvCacheFormat.SCALED_FP8.storage_dtype): + for n in (rows, 64): + t = replicated_zeros(mesh_device, n, width, dtype) + assert list(t.shape) == [1, 1, n, width] and t.dtype == dtype and t.layout == ttnn.ROW_MAJOR_LAYOUT + values = _all_chips(mesh_device, t) + assert values.shape[0] == mesh_device.get_num_devices() and bool((values == 0).all()), (dtype, n) + ttnn.deallocate(t) + # the state's ratio-1 KV tensor (window region + one row per token) spans several zero blocks + chunk, max_seq = 1024, 8192 + for fmt in V41PrefillState.FORMATS: + state = V41PrefillState(mesh_device, C, max_seq, chunk, [0, 2, 3, 20], kv_format=fmt) + kv = state.kv[20] + assert kv.shape[2] == state.geometry.kv_rows(1) and kv.shape[2] * kv.shape[3] > ZERO_BLOCK_BYTES + for t in (kv, state.kv[2], state.index_k[20], state.swa_scratch, state.window_carry[20]): + assert bool((_all_chips(mesh_device, t) == 0).all()), fmt diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_compressor.py b/models/demos/deepseek_v3_d_p/tests/v41/test_compressor.py new file mode 100644 index 000000000000..e3b8c738aca9 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_compressor.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 compressor (bead F2) vs the reference ``Compressor``: ratio 2 and 1, synthetic and real +weights, an odd valid length (partial group -> carry), chunk boundaries, determinism. Bar (G1): 0.999.""" + +import json +from pathlib import Path + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41.compressor import TtV41Compressor +from tests.ttnn.utils_for_testing import comp_pcc + +PCC = 0.999 +SEQ = 2048 +SNAPSHOT = Path.home() / ( + ".cache/huggingface/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277" +) + + +def _weights(source: str, layer: int, ratio: int, gen) -> dict: + if source == "synthetic": + w = {"wkv": torch.randn(C.HEAD_DIM, C.EMB_SIZE, generator=gen) * C.EMB_SIZE**-0.5} + if ratio > 1: + w["wgate"] = torch.randn(C.HEAD_DIM, C.EMB_SIZE, generator=gen) * C.EMB_SIZE**-0.5 + w["norm"] = 1 + 0.1 * torch.randn(C.HEAD_DIM, generator=gen) + return {k: v.to(torch.bfloat16) for k, v in w.items()} + from safetensors import safe_open + + if not (SNAPSHOT / "model.safetensors.index.json").is_file(): + pytest.skip("V4.1 checkpoint shards not downloaded") + index = json.loads((SNAPSHOT / "model.safetensors.index.json").read_text())["weight_map"] + keys = {"wkv": "wkv.weight", "norm": "norm.weight"} | ({"wgate": "wgate.weight"} if ratio > 1 else {}) + out = {} + for k, suffix in keys.items(): + name = f"layers.{layer}.attn.compressor.{suffix}" + with safe_open(SNAPSHOT / index[name], "pt") as f: + out[k] = f.get_tensor(name) + return out + + +def _reference(ratio, weights, x): + """Reference Compressor over x [1, L, dim] bf16 -> (rows [L//r, d], kv_state row of an odd tail or None).""" + args = v41.ModelArgs( + max_batch_size=1, + max_seq_len=SEQ, + dim=C.EMB_SIZE, + head_dim=C.HEAD_DIM, + compress_ratios=(ratio,), + norm_eps=C.RMS_NORM_EPS, + ) + with v41.set_dtype(torch.bfloat16): + comp = v41.Compressor(args, 0) + with torch.no_grad(): + comp.wkv.weight.copy_(weights["wkv"].to(comp.wkv.weight.dtype)) + if ratio > 1: + comp.wgate.weight.copy_(weights["wgate"].to(comp.wgate.weight.dtype)) + comp.norm.weight.copy_(weights["norm"].float()) + rows = comp(x, 0) + carry = comp.kv_state[0, 0].clone() if ratio > 1 and x.shape[1] % ratio else None + return rows[0], carry + + +@pytest.mark.parametrize("weights_source", ["synthetic", "real"]) +@pytest.mark.parametrize("layer", [2, 20], ids=["ratio2-L2", "ratio1-L20"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) +def test_v41_compressor(mesh_device, device_params, layer, weights_source): + ratio = C.compress_ratio(layer) + gen = torch.Generator().manual_seed(layer) + weights = _weights(weights_source, layer, ratio, gen) + shape, tp = tuple(mesh_device.shape), mesh_device.shape[1] + comp = TtV41Compressor(mesh_device, C, layer, weights) + x = torch.randn(1, SEQ, C.EMB_SIZE, generator=gen).to(torch.bfloat16) + + def run(tokens): + tt = ttnn.from_torch( + tokens[None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + latent, carry = comp(tt) + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + rows = down(latent)[0, 0, :, : C.HEAD_DIM] # replicated across TP + kv = None if carry is None else down(carry[0])[0, 0, :, : C.HEAD_DIM] + return rows, kv + + rows, kv = run(x[0]) + rows2, _ = run(x[0]) + assert torch.equal(rows, rows2), "compressor is not bit-identical across repeats" + + results = {} + ref_rows, _ = _reference(ratio, weights, x) + results["full"] = comp_pcc(ref_rows.float(), rows.float(), 0.0)[1] + + # odd valid length: rows of complete groups only, trailing token's projection is the carry + valid = SEQ - 3 + ref_odd, ref_carry = _reference(ratio, weights, x[:, :valid]) + results["odd_rows"] = comp_pcc(ref_odd.float(), rows[: valid // ratio].float(), 0.0)[1] + if ratio > 1: + results["odd_carry"] = comp_pcc(ref_carry.float(), kv[valid - 1].float(), 0.0)[1] + + # chunk boundary: two chunks of half the sequence give the single-shot rows + half = SEQ // 2 + c0, _ = run(x[0, :half]) + c1, _ = run(x[0, half:]) + results["chunked"] = comp_pcc(ref_rows.float(), torch.cat([c0, c1]).float(), 0.0)[1] + print(f"compressor L{layer} {weights_source}: {results}") + for key, value in results.items(): + assert value >= PCC, (key, results) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_dspark.py b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark.py new file mode 100644 index 000000000000..1d0e6468e6b5 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark.py @@ -0,0 +1,214 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 DSpark prefill seeding (beads 9.1 / 9.2) vs the reference oracle. + +The oracle runs the vendored reference single-shot (backbone, then ``forward_spec`` at start_pos 0) and captures +the block input streams of the tap layers, the concatenated taps (``main_hidden``) and the DSpark window rings. +The device gets the oracle's tap-layer streams, per chunk and padded with junk rows beyond the valid length, and +must reproduce: the taps (N6), main_x = main_norm(main_proj(taps)) on the rows it seeds (D1, G1 bar for new +linear ops 0.999), and the final rings (D2, 0.998), bit-identically on repeat. Rings slots a prompt never reaches +must stay zero. Cases: prompts shorter than the window, longer than it, and a chunked prompt whose last +``window`` positions span two chunks. + +"small" is the oracle's small schedule (one DSpark layer, taps of layers 4 and 5, window 16); "real" is V4.1 +layers 20, 36-39 with the three DSpark layers at real dims (window 128). Weights: synthetic, or ("real_ckpt") the +checkpoint's DSpark units ``mtp.0-2`` on the synthetic backbone of "real" (the backbone shards are not downloaded; +the backbone only produces the taps, which are teacher-forced from the oracle either way). +""" + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as o +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41.dspark import TtV41DSpark +from models.demos.deepseek_v3_d_p.tt.v41.weights import dequant_fp8_block +from tests.ttnn.utils_for_testing import comp_pcc + +MAIN_X_PCC = 0.999 +TAP_PCC = 0.999 +RING_PCC = 0.998 + +REAL_LAYERS = (20, 36, 37, 38, 39) +DSPARK_UNITS = tuple(f"mtp.{k}" for k in range(DeepSeekV41FlashConfig.NUM_DSPARK_LAYERS)) +REAL_PROMPTS = [(100, [100]), (300, [300]), (300, [256, 44])] # window 128: < window, > window, last 128 span two + +# (spec, prompt length, chunk lengths); every chunk but the last is a multiple of 2*32*sp (sp = 2) +CASES = { + "small": ( + lambda: o.small_spec(seq_len=256), + [(13, [13]), (100, [100]), (135, [128, 7])], # window 16: < window, > window, last 16 span two chunks + ), + "real": (lambda: o.real_spec(REAL_LAYERS, 512, dspark=True), REAL_PROMPTS), + "real_ckpt": ( + lambda: o.real_spec(REAL_LAYERS, 512, dspark=True, checkpoint=o.HF_SNAPSHOT, checkpoint_units=DSPARK_UNITS), + REAL_PROMPTS, + ), +} + + +def dspark_shards_present() -> bool: + """Whether the checkpoint shards of the DSpark units are downloaded.""" + import json + + index = o.HF_SNAPSHOT / "model.safetensors.index.json" + if not index.is_file(): + return False + weight_map = json.loads(index.read_text())["weight_map"] + files = {f for name, f in weight_map.items() if name.split(".")[0] == "mtp"} + return bool(files) and all((o.HF_SNAPSHOT / f).is_file() for f in files) + + +def _config(args: v41.ModelArgs): + """The config attributes TtV41DSpark reads, from the reference args (equal to the canonical ones at real + dims, checked in the test).""" + return type( + "DSparkTestConfig", + (DeepSeekV41FlashConfig,), + { + "EMB_SIZE": args.dim, + "HC_MULT": args.hc_mult, + "HEAD_DIM": args.head_dim, + "QK_ROPE_HEAD_DIM": args.rope_head_dim, + "SLIDING_WINDOW": args.window_size, + "RMS_NORM_EPS": args.norm_eps, + "ROPE_THETA": args.rope_theta, + "NUM_DSPARK_LAYERS": args.n_mtp_layers, + "DSPARK_TARGET_LAYER_IDS": args.dspark_target_layer_ids, + }, + ) + + +def _fp8(linear) -> torch.Tensor: + return dequant_fp8_block(linear.weight.detach(), linear.scale.detach()) + + +def _weights(model: v41.Transformer) -> dict: + stage0 = model.mtp[0] + return { + "main_proj": _fp8(stage0.main_proj), + "main_norm": stage0.main_norm.weight.detach(), + "layers": [{"wkv": _fp8(b.attn.wkv), "kv_norm": b.attn.kv_norm.weight.detach()} for b in model.mtp], + } + + +def _pack(x: torch.Tensor, tp: int) -> torch.Tensor: + """[S, hc, D] -> [1, 1, S, hc * D] with chip t's columns = its D/tp slice of every copy (mhc.py layout).""" + s, n, d = x.shape + return x.reshape(s, n, tp, d // tp).permute(0, 2, 1, 3).reshape(1, 1, s, n * d) + + +def _pcc(a, b) -> float: + return comp_pcc(a.float(), b.float(), 0.0)[1] + + +@pytest.mark.timeout(7200) +@pytest.mark.parametrize("dims", ["small", "real", "real_ckpt"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) +def test_v41_dspark_seeding(mesh_device, device_params, dims): + make_spec, prompts = CASES[dims] + if dims == "real_ckpt" and not dspark_shards_present(): + pytest.skip(f"DSpark checkpoint shards not in {o.HF_SNAPSHOT}") + spec = make_spec() + args = spec.args + cfg = _config(args) + if dims != "small": + for name in ("EMB_SIZE", "HEAD_DIM", "QK_ROPE_HEAD_DIM", "SLIDING_WINDOW", "RMS_NORM_EPS", "ROPE_THETA"): + assert getattr(cfg, name) == getattr(DeepSeekV41FlashConfig, name), name + tap_ids = [spec.layer_ids[p] for p in args.dspark_target_layer_ids] + if dims != "small": + assert tap_ids == list(DeepSeekV41FlashConfig.DSPARK_TARGET_LAYER_IDS) + model = o.build_reference(spec) + shape, (sp, tp) = tuple(mesh_device.shape), tuple(mesh_device.shape) + pad = 2 * 32 * sp + dim, window, n_taps = args.dim, args.window_size, len(tap_ids) + dspark = TtV41DSpark(mesh_device, cfg, _weights(model)) + gen = torch.Generator().manual_seed(3) + + def first_device(t) -> torch.Tensor: + return ttnn.to_torch(ttnn.get_device_tensors(t)[0])[0, 0] + + def all_devices_equal(t) -> bool: + views = [ttnn.to_torch(d) for d in ttnn.get_device_tensors(t)] + return all(torch.equal(views[0], v) for v in views[1:]) + + results, exact = {}, {} + for seq, chunks in prompts: + tag = f"S{seq}-" + "+".join(map(str, chunks)) + result = o.oracle(spec, o.random_tokens(spec, seq), model) + main_hidden = result["state"]["main_hidden"] # [S, n_taps * dim] + with v41.set_dtype(torch.bfloat16), torch.no_grad(): + ref_main_x = model.mtp[0].main_norm(model.mtp[0].main_proj(main_hidden[None]))[0] + + def chunk_taps(start, length): + padded = -(-length // pad) * pad + taps = [] + for lid in tap_ids: + x = torch.randn(padded, args.hc_mult, dim, generator=gen) * 100 # junk beyond the valid rows + x[:length] = result["blocks"][lid]["x_in"][start : start + length].float() + tt = ttnn.from_torch( + _pack(x, tp), + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + taps.append(dspark.tap(tt)) + return taps + + def run(check: bool): + rings, start, main_xs = dspark.new_rings(), 0, [] + for length in chunks: + taps = chunk_taps(start, length) + if check: + for i, t in enumerate(taps): + dev = ttnn.to_torch( + t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3)) + )[0, 0, :length] + ref = main_hidden[start : start + length, i * dim : (i + 1) * dim] + results[f"{tag}_c{start}_tap{tap_ids[i]}"] = _pcc(ref, dev) + main_x, a = dspark.project(taps, length) + end = min(main_x.shape[2], length - a) + main_xs.append(first_device(main_x)[:end]) + if check: + ref = ref_main_x[start + a : start + a + end] + results[f"{tag}_c{start}_main_x"] = _pcc(ref, main_xs[-1]) + exact[f"{tag}_c{start}_main_x_replicated"] = all_devices_equal(main_x) + dspark.seed(taps, start, length, rings, dspark.seed_rope(start, length)) + start += length + return [first_device(r) for r in rings], main_xs, rings + + rings, main_xs, tt_rings = run(check=True) + rings2, main_xs2, _ = run(check=False) + exact[f"{tag}_deterministic"] = all(torch.equal(a, b) for a, b in zip(rings + main_xs, rings2 + main_xs2)) + for k, ring in enumerate(rings): + ref = result["state"]["dspark_window"][k] + results[f"{tag}_ring{k}"] = _pcc(ref, ring) + exact[f"{tag}_ring{k}_replicated"] = all_devices_equal(tt_rings[k]) + if seq < window: # slots of positions the prompt never reached stay zero (rule 8: padding writes nothing) + exact[f"{tag}_ring{k}_unwritten_zero"] = bool((ring[seq:] == 0).all()) + exact_rows = (ring.float() == ref.float()).all(dim=-1).float().mean().item() + print(f"{tag} ring{k}: rows bit-equal to the reference {exact_rows:.3f}") + print(f"dspark {dims}: {results}") + print(f"dspark {dims} exact: {exact}") + for key, value in results.items(): + bar = RING_PCC if "_ring" in key else MAIN_X_PCC if key.endswith("main_x") else TAP_PCC + assert value >= bar, (key, value, bar) + for key, value in exact.items(): + assert value, key diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks.py b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks.py new file mode 100644 index 000000000000..983df34e45db --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The multi-block draft call's host inputs (``tt/v41/dspark_blocks.py``), on the workstation: no ttnn, no +device. ``python -m pytest --noconftest -p no:cacheprovider +models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks.py`` +(the tree's conftest opens the cluster at collection).""" + +import importlib.util +import pathlib + +import pytest +import torch + +_PATH = pathlib.Path(__file__).resolve().parents[2] / "tt" / "v41" / "dspark_blocks.py" +_SPEC = importlib.util.spec_from_file_location("dspark_blocks", _PATH) +db = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(db) # the module alone: tt.v41's package __init__ imports ttnn + + +def _single_reference(token: int, p: int, ring_slots: int, write: bool, tap_row: int) -> dict: + """The one-block inputs as ``TtDSparkDrafter.host_inputs`` built them (the + reference's ``get_dspark_topk_idxs``: the written ring slots, then the block).""" + ids = torch.full((db.ROWS,), db.NOISE, dtype=torch.int32) + ids[0] = token + sel = torch.zeros(db.ROWS, db.ROWS) + ring_idx = torch.full((db.ROWS,), db.SENTINEL, dtype=torch.int32) + if write: + sel[0, tap_row] = 1.0 + ring_idx[0] = p % ring_slots + valid = min(db.RING, p + 1) + rows = [db.SENTINEL] * db.INDEX_WIDTH + if ring_slots == db.RING: + rows[:valid] = range(valid) + else: + rows[:valid] = [q % ring_slots for q in range(p - valid + 1, p + 1)] + rows[valid : valid + db.BLOCK] = range(ring_slots, ring_slots + db.BLOCK) + index = torch.tensor(rows, dtype=torch.int32).view(1, -1).expand(db.ROWS, -1).contiguous() + return {"ids": ids, "sel": sel, "ring_idx": ring_idx, "index": index} + + +@pytest.mark.parametrize("ring_slots", [128, 160]) +@pytest.mark.parametrize("p", [5, 127, 128, 300, 1054655]) +def test_one_block_matches_the_single_call(ring_slots, p): + """One entry of user 0 reproduces the single call's token, selector, ring index and block rows' index rows + (the rows past the block are noise rows whose index rows were block 0's before and are the sentinel now: + their outputs are never read).""" + for write in (False, True): + got = db.rank_inputs([{"user": 0, "token": 7, "position": p, "taps": [(3, p)] if write else []}], ring_slots, 1) + ref = _single_reference(7, p, ring_slots, write, 3) + assert torch.equal(got["ids"], ref["ids"]) + assert torch.equal(got["sel"], ref["sel"]) + assert torch.equal(got["ring_idx"], ref["ring_idx"]) + assert torch.equal(got["index"][: db.BLOCK], ref["index"][: db.BLOCK]) + valid = min(db.RING, p + 1) + assert torch.equal(got["index"][db.BLOCK :, :valid], ref["index"][db.BLOCK :, :valid]) + assert (got["index"][db.BLOCK :, valid:] == db.SENTINEL).all() + assert torch.equal(got["block_pos"], torch.arange(p + 1, p + 1 + db.ROWS)) + assert int(got["main_pos"][0]) == (p if write else 0) + + +def test_four_blocks_four_rings(): + """Four slots of a rank at different frontiers with different accepted counts: each block's token on its + first row, its taps on its rows, its ring's rows in its index rows, its own block rows after them, the + sentinel elsewhere; nothing of one slot in another's rows.""" + ring_slots, users = 160, 4 + entries = [ + {"user": 0, "token": 11, "position": 300, "taps": [(0, 298), (1, 299), (2, 300)]}, + {"user": 1, "token": 12, "position": 1000, "taps": [(4, 1000)]}, + {"user": 2, "token": 13, "position": 64, "taps": []}, + {"user": 3, "token": 14, "position": 2047, "taps": [(12, 2046), (13, 2047)]}, + ] + t = db.rank_inputs(entries, ring_slots, users) + block_base = users * ring_slots + for a, e in enumerate(entries): + u, p, r0 = e["user"], e["position"], db.BLOCK * a + assert int(t["ids"][r0]) == e["token"] + assert all(int(t["ids"][r0 + i]) == db.NOISE for i in range(1, db.BLOCK)) + for i, (tap_row, q) in enumerate(e["taps"]): + assert t["sel"][r0 + i].sum() == 1 and t["sel"][r0 + i, tap_row] == 1 + assert int(t["ring_idx"][r0 + i]) == u * ring_slots + q % ring_slots + assert int(t["main_pos"][r0 + i]) == q + for i in range(len(e["taps"]), db.BLOCK): + assert t["sel"][r0 + i].sum() == 0 and int(t["ring_idx"][r0 + i]) == db.SENTINEL + valid = min(db.RING, p + 1) + want = [u * ring_slots + q % ring_slots for q in range(p - valid + 1, p + 1)] + want += list(range(block_base + r0, block_base + r0 + db.BLOCK)) + for i in range(db.BLOCK): + row = t["index"][r0 + i].tolist() + assert row[: len(want)] == want + assert all(v == db.SENTINEL for v in row[len(want) :]) + # the user's ring region alone, then the block rows + assert all(u * ring_slots <= v < (u + 1) * ring_slots for v in row[:valid]) + assert t["block_pos"][r0 : r0 + db.BLOCK].tolist() == list(range(p + 1, p + 1 + db.BLOCK)) + # the rows past the blocks: noise, no write, block 0's window alone (never an all-sentinel row) + window = t["index"][0].tolist()[: db.RING] + for r in range(db.BLOCK * len(entries), db.ROWS): + assert int(t["ids"][r]) == db.NOISE and int(t["ring_idx"][r]) == db.SENTINEL + assert t["sel"][r].sum() == 0 + row = t["index"][r].tolist() + assert row[: db.RING] == window and all(v == db.SENTINEL for v in row[db.RING :]) + assert t["index"].max() < block_base + db.ROWS + + +def test_mesh_inputs_stack_the_ranks(): + a = [{"user": 0, "token": 1, "position": 200, "taps": [(0, 200)]}] + b = [{"user": 3, "token": 2, "position": 400, "taps": []}] + t = db.mesh_inputs([a, b, a, b], 160, 4) + assert t["ids"].shape == (4, db.ROWS) and t["index"].shape == (4, db.ROWS, db.INDEX_WIDTH) + assert torch.equal(t["ids"][0], t["ids"][2]) and torch.equal(t["index"][1], t["index"][3]) + assert int(t["ids"][1, 0]) == 2 and int(t["ring_idx"][1, 0]) == db.SENTINEL + assert int(t["ring_idx"][0, 0]) == 200 % 160 + + +def test_read_drafts(): + ids = torch.arange(db.ROWS) + assert db.read_drafts(ids, 2) == [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]] + + +def test_limits(): + with pytest.raises(AssertionError): # allow-pytest.raises: host-only test uses --noconftest + db.rank_inputs([{"user": 0, "token": 1, "position": 10, "taps": []}] * 7, 160, 7) + with pytest.raises(AssertionError): # allow-pytest.raises: host-only test uses --noconftest + db.rank_inputs([{"user": 2, "token": 1, "position": 10, "taps": []}], 160, 2) + with pytest.raises(AssertionError): # allow-pytest.raises: host-only test uses --noconftest + db.rank_inputs([{"user": 0, "token": 1, "position": 10, "taps": [(0, 11)]}], 160, 1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks_mesh.py b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks_mesh.py new file mode 100644 index 000000000000..77905461d5de --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks_mesh.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The one-pass ``dspark_blocks.mesh_inputs`` against the per-rank reference, on the +workstation: no ttnn, no device. ``python -m pytest --noconftest -p no:cacheprovider +models/demos/deepseek_v3_d_p/tests/v41/test_dspark_blocks_mesh.py``.""" + +import importlib.util +import pathlib +import random + +import torch + +_PATH = pathlib.Path(__file__).resolve().parents[2] / "tt" / "v41" / "dspark_blocks.py" +_SPEC = importlib.util.spec_from_file_location("dspark_blocks", _PATH) +db = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(db) # the module alone: tt.v41's package __init__ imports ttnn + + +def test_mesh_inputs_match_the_reference(): + """The one-pass ``mesh_inputs`` equals the per-rank ``mesh_inputs_reference`` on random entries (every + ring + width, block count and position regime, taps of every length, some ranks without taps).""" + rng = random.Random(181) + for ring_slots in (128, 160): + for blocks in (1, 2, 3, 4): + for pmax in (5, 200, 70000): + for _ in range(10): + entries = [] + for _r in range(8): + es = [] + for u in rng.sample(range(4), blocks): + p = rng.randint(0, pmax) + a = rng.randint(0, 4) + taps = [(rng.randint(0, db.ROWS - 1), p - a + j) for j in range(a + 1)] + taps = [(t, q) for t, q in taps if 0 <= q <= p] if rng.random() < 0.9 else [] + es.append({"user": u, "token": rng.randint(0, 100000), "position": p, "taps": taps}) + entries.append(es) + got, ref = db.mesh_inputs(entries, ring_slots, 4), db.mesh_inputs_reference(entries, ring_slots, 4) + assert got.keys() == ref.keys() + for k in ref: + assert got[k].dtype == ref[k].dtype and got[k].shape == ref[k].shape, k + assert torch.equal(got[k], ref[k]), (k, ring_slots, blocks, pmax) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_drafter.py b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_drafter.py new file mode 100644 index 000000000000..5f0241b53eb0 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_drafter.py @@ -0,0 +1,307 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The DSpark drafter on the mesh (``tt/v41/dspark_drafter.py``) against the vendored reference (speculative +decode device gate). + +The reference is ``real_spec`` with the backbone of layers 20, 36-39 synthetic and the three ``mtp.*`` units +from the +checkpoint (as ``test_dspark.py``'s ``real_ckpt``), greedy. It prefills ``V41_DSD_PROMPT`` tokens of real text +and seeds +its rings (``forward_spec`` at start_pos 0); the device drafter is built from the checkpoint through the +loader +(the routed experts converted into the drafter's own cache directory on the first run) with the reference's +tied +embedding / head, and takes the reference's rings. Then, for ``V41_DSD_STEPS`` positions, both sides run one +draft +call on the reference's tap of the position, teacher-forced on the text's next token, and the test compares: +stage +0's ``main_x`` and every stage's ring write (bit-exact rate and PCC), every stage's output streams on the +block's +rows (PCC), the five rows' logits after the Markov loop (PCC) and the drafted ids (exact matches), plus the +determinism of a repeated call. It reports the draft call's wall time (eager; the Markov loop on the host and +the +logits readback apart) and the DRAM a chip before / after the drafter: ``V41_DSD_OUT`` takes the JSON record. +""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import WEIGHT_CACHE, routed_expert_dtype +from models.demos.deepseek_v3_d_p.tt.v41.dspark_drafter import DSPARK_MARKOV, RING, ROWS, TtDSparkDrafter +from models.demos.deepseek_v3_d_p.tt.v41.head import TtV41Embedding +from models.demos.deepseek_v3_d_p.tt.v41.weights import resolve_checkpoint +from tests.ttnn.utils_for_testing import comp_pcc + +REAL_LAYERS = (20, 36, 37, 38, 39) +DSPARK_UNITS = tuple(f"mtp.{k}" for k in range(C.NUM_DSPARK_LAYERS)) +TRACE_REGION = 256 * 1024**2 + +MAIN_X_PCC = 0.999 +RING_PCC = 0.998 +# the stages' streams and the logits against the bf16 reference with bfp8 / bfp4 routed experts (measured: +# stage 2 at 0.95 to 0.97 with bfp4, the logits 0.98 to 0.9999; the drafted ids agree) +STREAM_PCC = {"BFLOAT8_B": 0.97, "BFLOAT4_B": 0.94} +LOGITS_PCC = 0.97 + + +def _pcc(a, b) -> float: + return float(comp_pcc(a.float(), b.float(), 0.0)[1]) + + +def _dram(mesh_device) -> dict: + """A chip's DRAM: bytes allocated, bytes free (the live margin) and the largest free block, from the + allocator.""" + try: + view = ttnn.get_memory_view(mesh_device, ttnn.BufferType.DRAM) + banks = int(view.num_banks) + return { + "allocated": int(view.total_bytes_allocated_per_bank) * banks, + "free": int(view.total_bytes_free_per_bank) * banks, + "largest_free_per_bank": int(view.largest_contiguous_bytes_free_per_bank), + "banks": banks, + } + except Exception as e: + logger.warning(f"DRAM statistics unavailable: {e}") + return {} + + +def _first(t) -> torch.Tensor: + return ttnn.to_torch(ttnn.get_device_tensors(t)[0])[0, 0] + + +def _streams_to_host(t, mesh_device, hc: int, hidden: int) -> torch.Tensor: + """Device streams [1, 1, ROWS, hc * hidden/tp] fp32 per chip (chip t: its hidden slice of every copy) -> + [ROWS, + hc, hidden] from mesh row 0.""" + sp, tp = tuple(mesh_device.shape) + shards = ttnn.get_device_tensors(t) + pieces = [ttnn.to_torch(shards[j])[0, 0] for j in range(tp)] # mesh row 0's chips + x = torch.cat(pieces, dim=-1) # [ROWS, tp * hc * hidden/tp] in (chip, copy, j) order + return x.reshape(ROWS, tp, hc, hidden // tp).permute(0, 2, 1, 3).reshape(ROWS, hc, hidden) + + +@pytest.mark.timeout(int(os.environ.get("V41_DSD_TIMEOUT", 3600))) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=TRACE_REGION), indirect=True) +def test_dspark_drafter(mesh_device, device_params): + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + prompt = int(os.environ.get("V41_DSD_PROMPT", 160)) + steps = int(os.environ.get("V41_DSD_STEPS", 3)) + timed = int(os.environ.get("V41_DSD_TIMED", 5)) + stages = int(os.environ.get("V41_DSD_STAGES", C.NUM_DSPARK_LAYERS)) + assert prompt >= RING, "the reference attends min(128, p + 1) ring slots; keep the ring full" + sp, tp = tuple(mesh_device.shape) + dtype = routed_expert_dtype() + record = { + "prompt": prompt, + "steps": steps, + "stages": stages, + "expert_dtype": dtype.name, + "mesh": [sp, tp], + "flags": {k: v for k, v in os.environ.items() if k.startswith(("V41_", "TT_V41_"))}, + } + + # --- the reference, prefilled and seeded --- + t0 = time.time() + spec = orc.real_spec(REAL_LAYERS, 512, dspark=True, checkpoint=ckpt.root, checkpoint_units=DSPARK_UNITS) + model = orc.build_reference(spec) + model.temperature = 0.0 + for block in model.mtp: + block.temperature = 0.0 + # the tied tables go to the device in bf16: hold the reference to the same values (a synthetic unit's are + # fp32) + model.head.weight.data = model.head.weight.data.to(torch.bfloat16).float() + model.embed.weight.data = model.embed.weight.data.to(torch.bfloat16).to(model.embed.weight.dtype) + record["reference_build_s"] = round(time.time() - t0, 1) + tokens = orc.text_tokens(prompt + steps + 2) + t0 = time.time() + with torch.inference_mode(), v41.set_dtype(torch.bfloat16): + out_ids, _, main_hidden = model(tokens[:, :prompt], 0) + model.forward_spec(out_ids, main_hidden, 0) + record["reference_prefill_s"] = round(time.time() - t0, 1) + rings = [block.attn.window_kv_cache[0].clone() for block in model.mtp] + + captured = {} + + def stage_hook(k): + def hook(module, args, output): + captured[f"stage{k}"] = output[0].detach().clone() # [1, block, hc, dim] bf16 + + return hook + + def main_x_hook(module, args, output): + captured["main_x"] = output.detach().clone() # [1, 1, dim] bf16 + + for k, block in enumerate(model.mtp): + block.register_forward_hook(stage_hook(k)) + model.mtp[0].main_norm.register_forward_hook(main_x_hook) + + # --- the device drafter --- + record["dram_before"] = _dram(mesh_device) + t0 = time.time() + embedding = TtV41Embedding(mesh_device, C, model.embed.weight.detach().to(torch.bfloat16)) + record["dram_after_embedding"] = _dram(mesh_device) + drafter = TtDSparkDrafter( + mesh_device, + C, + ckpt, + embedding, + model.head.weight.detach().to(torch.bfloat16), + WEIGHT_CACHE, + dtype, + stages=stages, + ) + record["drafter_build_s"] = round(time.time() - t0, 1) + record["drafter_build_detail_s"] = drafter.build_s + record["cache_dir"] = str(drafter.cache_dir) + record["dram_after_drafter"] = _dram(mesh_device) + if record["dram_after_drafter"] and record["dram_after_embedding"]: + # the stages, the seeder and the head (the tied embedding is the backbone's in serving, the head too) + record["drafter_dram_bytes"] = ( + record["dram_after_drafter"]["allocated"] - record["dram_after_embedding"]["allocated"] + ) + drafter.load_rings(rings[:stages]) + hc, hidden = C.HC_MULT, C.EMB_SIZE + + results, exact, per_step = {}, {}, [] + walls = [] + for i in range(steps): + p = prompt + i + tok = int(tokens[0, p + 1]) + t0 = time.time() + with torch.inference_mode(), v41.set_dtype(torch.bfloat16): + _, _, main_hidden = model(tokens[:, p : p + 1], p) + ref_out, ref_logits, _ = model.forward_spec(tokens[0, p + 1 : p + 2], main_hidden, p) + ref_s = time.time() - t0 + ref_rings = [block.attn.window_kv_cache[0, p % RING].clone() for block in model.mtp] + + inputs = drafter.to_device(drafter.host_inputs(tok, main_hidden[0, 0], p)) + stage_out = {} + + def on_stage(k, x): + stage_out[k] = _streams_to_host(x, mesh_device, hc, hidden) + + ttnn.synchronize_device(mesh_device) + t0 = time.time() + if DSPARK_MARKOV == "device": + ids_dev, logits_dev, main_x = drafter.draft_ids(inputs, on_stage=on_stage) + device_ids = [int(v) for v in ttnn.to_torch(ttnn.get_device_tensors(ids_dev)[0]).flatten()[:5]] + else: + logits_dev, main_x = drafter.draft(inputs, on_stage=on_stage) + device_ids = None + rows = drafter.logits_rows(logits_dev) + wall = time.time() - t0 + t0 = time.time() + out, rows = drafter.markov_loop(tok, rows) + markov_s = time.time() - t0 + + tag = f"p{p}" + dev_main_x = _first(main_x)[0] + results[f"{tag}_main_x"] = _pcc(captured["main_x"][0, 0], dev_main_x) + exact[f"{tag}_main_x_bits"] = float((captured["main_x"][0, 0].float() == dev_main_x.float()).float().mean()) + for k in range(stages): + dev_ring = drafter.ring_rows(k)[p % RING] + results[f"{tag}_ring{k}"] = _pcc(ref_rings[k], dev_ring) + exact[f"{tag}_ring{k}_bits"] = float((ref_rings[k].float() == dev_ring.float()).float().mean()) + ref_x = captured[f"stage{k}"][0] # [block, hc, dim] + results[f"{tag}_stage{k}"] = _pcc(ref_x, stage_out[k][: ref_x.shape[0]]) + if stages == C.NUM_DSPARK_LAYERS: + results[f"{tag}_logits"] = _pcc(ref_logits[0], rows) + ref_ids = ref_out[0, 1:].tolist() + dev_ids = out[1:].tolist() + exact[f"{tag}_ids"] = sum(a == b for a, b in zip(ref_ids, dev_ids)) / len(ref_ids) + exact[f"{tag}_first_id"] = float(ref_ids[0] == dev_ids[0]) + if device_ids is not None: # the in-graph Markov loop against the host one (bf16 candidates: ties) + exact[f"{tag}_device_loop_ids"] = sum(a == b for a, b in zip(device_ids, dev_ids)) / len(dev_ids) + else: + ref_ids, dev_ids = [], [] + entry = { + "p": p, + "token": tok, + "ref_s": round(ref_s, 2), + "draft_wall_s": round(wall, 4), + "markov_host_s": round(markov_s, 4), + "ref_ids": ref_ids, + "dev_ids": dev_ids, + "device_loop_ids": device_ids, + } + per_step.append(entry) + logger.info(f"drafter step {json.dumps(entry)}") + for t in inputs.values(): + for u in t if isinstance(t, tuple) else (t,): + ttnn.deallocate(u) + + # --- determinism and the timed calls (the last position again) --- + p = prompt + steps - 1 + tok = int(tokens[0, p + 1]) + inputs = drafter.to_device(drafter.host_inputs(tok, main_hidden[0, 0], p)) + ids = [] + for _ in range(2): + logits_dev, _ = drafter.draft(inputs) + out, _ = drafter.markov_loop(tok, drafter.logits_rows(logits_dev)) + ids.append(out.tolist()) + exact["deterministic"] = float(ids[0] == ids[1]) + for _ in range(timed): + ttnn.synchronize_device(mesh_device) + t0 = time.time() + logits_dev, _ = drafter.draft(inputs) + ttnn.synchronize_device(mesh_device) + t1 = time.time() + rows = drafter.logits_rows(logits_dev) + t2 = time.time() + drafter.markov_loop(tok, rows) + t3 = time.time() + entry = {"device_wall_ms": (t1 - t0) * 1e3, "readback_ms": (t2 - t1) * 1e3, "markov_ms": (t3 - t2) * 1e3} + if DSPARK_MARKOV == "device": + ttnn.synchronize_device(mesh_device) + t0 = time.time() + ids_dev, logits_dev, _ = drafter.draft_ids(inputs) + ttnn.synchronize_device(mesh_device) + t1 = time.time() + ttnn.to_torch(ttnn.get_device_tensors(ids_dev)[0]) + t2 = time.time() + ttnn.deallocate(logits_dev) + entry["device_loop_wall_ms"], entry["device_loop_readback_ms"] = (t1 - t0) * 1e3, (t2 - t1) * 1e3 + walls.append(entry) + record["timed"] = [{k: round(v, 3) for k, v in w.items()} for w in walls] + record["draft_device_wall_ms_min"] = round(min(w["device_wall_ms"] for w in walls), 3) + record["draft_device_wall_ms_median"] = round(sorted(w["device_wall_ms"] for w in walls)[len(walls) // 2], 3) + record["readback_ms_min"] = round(min(w["readback_ms"] for w in walls), 3) + record["markov_ms_min"] = round(min(w["markov_ms"] for w in walls), 3) + record["dram_after_steps"] = _dram(mesh_device) + record["steps_detail"] = per_step + record["pcc"] = {k: round(v, 6) for k, v in results.items()} + record["exact"] = exact + logger.info("V41_DSD_RESULT " + json.dumps(record)) + out_path = os.environ.get("V41_DSD_OUT") + if out_path: + with open(out_path, "w") as f: + json.dump(record, f, indent=1) + print(f"dspark drafter pcc: {record['pcc']}") + print(f"dspark drafter exact: {exact}") + print(f"dspark drafter draft call: {record['timed']}") + for key, value in results.items(): + bar = ( + MAIN_X_PCC + if key.endswith("main_x") + else RING_PCC + if "_ring" in key + else LOGITS_PCC + if key.endswith("logits") + else STREAM_PCC[dtype.name] + ) + assert value >= bar, (key, value, bar) + assert exact["deterministic"], "two draft calls on the same inputs differ" diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_weights.py b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_weights.py new file mode 100644 index 000000000000..e92c1ddc360b --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_dspark_weights.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The DSpark drafter's weight loader (``tt/v41/dspark_weights.py``) against the checkpoint, CPU only. + +No device: run with ``python -m pytest --noconftest -p no:cacheprovider tests/v41/test_dspark_weights.py`` +from the +model's root (the tree's conftest opens the cluster at collection) or as a script. The schema test needs the +checkpoint's index only; the round-trip tests read one expert and two projections of stage 0 from shard 44 and +the Markov tables from shard 46 (``TT_V41_CHECKPOINT`` or the HF cache, else skipped) and check, against a +slow +re-dequantization written here from the format's definition: the dequantized values bit for bit, the per-chip +shards (expert placement, TP splits, the main projection's reorder) as slices of the checkpoint tensor, and +the +bfp8 round trip through a host ttnn tensor (what the device holds) within the block format's error. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest +import torch + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as cfg +from models.demos.deepseek_v3_d_p.tt.v41 import dspark_weights as dw +from models.demos.deepseek_v3_d_p.tt.v41.weights import ( + CHECKPOINT_REVISION, + INDEX_FILE, + V41Checkpoint, + resolve_checkpoint, +) + +MESH = (8, 4) # (sp, tp): the Galaxy mesh of the served model +EXPERTS_PER_CHIP = dw.DSPARK_ROUTED_EXPERTS // (MESH[0] * MESH[1]) + + +def _checkpoint(need_shards: bool) -> V41Checkpoint: + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("no checkpoint (TT_V41_CHECKPOINT or the HF cache)") + if need_shards: + files = {ckpt.weight_map[n] for n in dw.stage_tensor_names(cfg, 0)} + if missing := sorted(f for f in files if not (ckpt.root / f).is_file()): + pytest.skip(f"DSpark shards not downloaded: {missing}") + return ckpt + + +def _slow_fp8_block(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + """The format's definition, element by element over blocks: value = e4m3 * 2^e8m0 of its 32x32 block.""" + out = torch.empty(weight.shape, dtype=torch.float32) + s = scale.float() + for i in range(0, weight.shape[0], 32): + for j in range(0, weight.shape[1], 32): + out[i : i + 32, j : j + 32] = weight[i : i + 32, j : j + 32].float() * s[i // 32, j // 32] + return out.to(torch.bfloat16) + + +def _slow_mxfp4_rows(packed: torch.Tensor, scale: torch.Tensor, rows: range) -> torch.Tensor: + """MXFP4 rows ``rows``: nibble ``2j`` low, ``2j + 1`` high; E2M1 by its bit fields; one scale per 32 + elements.""" + table = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] + out = torch.empty(len(rows), packed.shape[1] * 2, dtype=torch.float32) + raw = packed.view(torch.uint8) + for r_out, r in enumerate(rows): + row = raw[r].tolist() + s = scale[r].float().tolist() + for j, byte in enumerate(row): + for half, code in enumerate((byte & 0x0F, byte >> 4)): + mag = table[code & 0x7] * (-1.0 if code & 0x8 else 1.0) + col = 2 * j + half + out[r_out, col] = mag * s[col // 32] + return out.to(torch.bfloat16) + + +def _pcc(a: torch.Tensor, b: torch.Tensor) -> float: + a, b = a.float().flatten(), b.float().flatten() + return float(torch.corrcoef(torch.stack([a, b]))[0, 1]) + + +def _bfp8_roundtrip(t: torch.Tensor) -> torch.Tensor: + """``t`` [rows, cols] bf16 -> the values a bfp8 tiled device tensor would hold (host ttnn, no device).""" + ttnn = pytest.importorskip("ttnn") + tt = ttnn.from_torch(t[None, None], dtype=ttnn.bfloat8_b, layout=ttnn.TILE_LAYOUT) + return ttnn.to_torch(tt)[0, 0].to(torch.bfloat16) + + +def test_stage_schema_against_index(): + """Every name the loader reads exists in the checkpoint index, with the stored dtype the schema expects, + and + the three stages differ only by the stage-specific heads (2,401 tensors in all).""" + ckpt = _checkpoint(need_shards=False) + assert ckpt.root.name == CHECKPOINT_REVISION or (ckpt.root / INDEX_FILE).is_file() + total = 0 + for stage in range(cfg.NUM_DSPARK_LAYERS): + names = dw.stage_tensor_names(cfg, stage) + assert len(names) == len(set(names)) + missing = [n for n in names if n not in ckpt] + assert not missing, missing[:5] + in_index = [n for n in ckpt.weight_map if n.startswith(dw.stage_prefix(stage))] + assert sorted(in_index) == sorted(names), (stage, len(in_index), len(names)) + total += len(names) + assert total == 2401 + assert dw.dspark_layer_id(0) == 40 and cfg.compress_ratio(dw.dspark_layer_id(2)) == 0 + + +def test_expert_roundtrip(): + """Stage 0, expert 0: the loader's dequantized w1 / w2 / w3 equal the slow definition on sampled rows; the + expert's chip under EP-32 is (row 0, column 0, local 0) and expert 127's (7, 3, 3); bfp8 keeps the values + within the block format (PCC > 0.999, every value within 1/64 of its tile row's largest).""" + ckpt = _checkpoint(need_shards=True) + (expert,) = dw.load_stage_experts(ckpt, 0, cfg, experts=range(0, 1)) + stem = f"{dw.stage_prefix(0)}ffn.experts.0" + raw = ckpt.read([f"{stem}.{w}.{p}" for w in ("w1", "w2", "w3") for p in ("weight", "scale")]) + for key, w, down in dw._EXPERT_PROJ: + t = expert[key] + assert t.dtype == torch.bfloat16 + assert tuple(t.shape) == ( + (cfg.EMB_SIZE, cfg.MOE_INTERMEDIATE_SIZE) if down else (cfg.MOE_INTERMEDIATE_SIZE, cfg.EMB_SIZE) + ) + rows = range(0, t.shape[0], max(1, t.shape[0] // 7)) + slow = _slow_mxfp4_rows(raw[f"{stem}.{w}.weight"], raw[f"{stem}.{w}.scale"], rows) + assert torch.equal(t[list(rows)], slow), key + assert dw.expert_chip(0, *MESH, EXPERTS_PER_CHIP) == (0, 0, 0) + assert dw.expert_chip(127, *MESH, EXPERTS_PER_CHIP) == (7, 3, 3) + assert dw.expert_chip(8 * 4 + 1, *MESH, EXPERTS_PER_CHIP) == (0, 1, 1) # chip 8 = group 1, chip 0; local 1 + w1 = expert["gate_proj"].T.contiguous() # the converter stacks w.T: [in, out] per expert + rt = _bfp8_roundtrip(w1) + assert _pcc(w1, rt) > 0.999 + step = w1.float().abs().reshape(-1, 32, 32).amax(dim=2, keepdim=True).expand(-1, 32, 32).reshape(w1.shape) / 64 + assert ((w1.float() - rt.float()).abs() <= step + 1e-6).all() + + +def test_projection_roundtrip(): + """Stage 0's ``wq_a`` [1280, 5120] and ``wq_b`` [32768, 1280]: the loader's dequantization equals the slow + definition; the TP-4 row and column shards are slices of the transposed tensor; bfp8 within the format.""" + ckpt = _checkpoint(need_shards=True) + dense = dw.load_stage_dense(ckpt, 0, cfg) + attn = dense["attn"] + raw = ckpt.read([f"{dw.stage_prefix(0)}attn.{n}.{p}" for n in ("wq_a", "wq_b") for p in ("weight", "scale")]) + for name in ("wq_a", "wq_b"): + t = attn[name] + slow = _slow_fp8_block( + raw[f"{dw.stage_prefix(0)}attn.{name}.weight"], raw[f"{dw.stage_prefix(0)}attn.{name}.scale"] + ) + assert torch.equal(t, slow), name + rows = dw.split_tp_rows(attn["wq_a"], MESH[1]) + assert [tuple(r.shape) for r in rows] == [(cfg.EMB_SIZE // MESH[1], cfg.Q_LORA_RANK)] * MESH[1] + assert torch.equal(torch.cat(rows, dim=0), attn["wq_a"].T) + cols = dw.split_tp_cols(attn["wq_b"], MESH[1]) + heads_per_chip = cfg.NUM_ATTENTION_HEADS // MESH[1] + assert [tuple(c.shape) for c in cols] == [(cfg.Q_LORA_RANK, heads_per_chip * cfg.HEAD_DIM)] * MESH[1] + assert torch.equal(torch.cat(cols, dim=1), attn["wq_b"].T) + rt = _bfp8_roundtrip(rows[1]) + assert _pcc(rows[1], rt) > 0.999 + assert tuple(dense["main_proj"].shape) == (cfg.EMB_SIZE, 3 * cfg.EMB_SIZE) + shards = dw.main_proj_shards(dense["main_proj"], 3, MESH[1]) + full_t = dense["main_proj"].T # [3 * hidden, hidden], rows (tap, j) + h, q = cfg.EMB_SIZE, cfg.EMB_SIZE // MESH[1] + for t_idx, shard in enumerate(shards): + assert tuple(shard.shape) == (3 * q, h) + for tap in range(3): # shard t holds tap k's rows [t * q, (t + 1) * q) in tap order + assert torch.equal(shard[tap * q : (tap + 1) * q], full_t[tap * h + t_idx * q : tap * h + (t_idx + 1) * q]) + + +def test_markov_tables_and_seeding_dict(): + """The last stage's Markov tables are bf16 [vocab, 256] (vocab / tp = 32,320 rows a chip when + column-parallel + as the tied head); ``seeding_weights`` is ``TtV41DSpark``'s dict and equals the stage loaders' tensors.""" + ckpt = _checkpoint(need_shards=True) + last = cfg.NUM_DSPARK_LAYERS - 1 + dense = dw.load_stage_dense(ckpt, last, cfg) + for name in ("embed", "head"): + t = dense["markov"][name] + assert t.dtype == torch.bfloat16 and tuple(t.shape) == (cfg.VOCAB_SIZE, dw.DSPARK_MARKOV_RANK) + assert cfg.VOCAB_SIZE % MESH[1] == 0 and cfg.VOCAB_SIZE // MESH[1] == 32320 + assert tuple(dense["confidence"].shape) == (1, cfg.EMB_SIZE + dw.DSPARK_MARKOV_RANK) + assert tuple(dense["norm"].shape) == (cfg.EMB_SIZE,) + seed = dw.seeding_weights(ckpt, cfg) + assert len(seed["layers"]) == cfg.NUM_DSPARK_LAYERS + assert torch.equal(seed["layers"][last]["wkv"], dense["attn"]["wkv"]) + assert torch.equal(seed["layers"][last]["kv_norm"], dense["attn"]["kv_norm"]) + stage0 = dw.load_stage_dense(ckpt, 0, cfg) + assert torch.equal(seed["main_proj"], stage0["main_proj"]) and torch.equal(seed["main_norm"], stage0["main_norm"]) + assert torch.equal(seed["layers"][0]["wkv"], stage0["attn"]["wkv"]) + + +def test_cache_dir_is_namespaced(tmp_path: Path): + """The drafter's cache directory is its own, keyed by dtype and mesh, never the backbone's ``real-`` one.""" + a = dw.dspark_cache_dir(tmp_path, MESH, "BFLOAT8_B") + b = dw.dspark_cache_dir(tmp_path, MESH, "BFLOAT4_B") + c = dw.dspark_cache_dir(tmp_path, (2, 4), "BFLOAT8_B") + assert a.parent == tmp_path and a.name.startswith("dspark-") and a.name.endswith("-mesh8x4") + assert len({a, b, c}) == 3 and c.name.endswith("-mesh2x4") + assert a == dw.dspark_cache_dir(tmp_path, MESH, "BFLOAT8_B") # deterministic + + +if __name__ == "__main__": + sys.exit(pytest.main(["--noconftest", "-p", "no:cacheprovider", "-v", os.path.abspath(__file__)])) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_engram_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_engram_v41.py new file mode 100644 index 000000000000..b2950e08560a --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_engram_v41.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 Engram (bead F5): host hash + lookup, device wkv + gated stream update, vs the reference. + +Expected results come from ``engram_reference.engram_case`` (the vendored reference's ``NgramHashState``, +``ParallelEngramEmbedding`` on synthetic rows and ``Engram.forward``, cached on disk; precompute with +``python -m models.demos.deepseek_v3_d_p.tests.v41.engram_reference``). Checks: + +* host hash ids == reference, single-shot and chunked with the carried history (chunks shorter than an n-gram, + an image span across a chunk boundary); the history is the last 3 compressed ids; +* host lookup rows == reference lookup (bit-exact); device rows (packed rows uploaded, or a device-resident + row-sharded table gathered by id; both dequantized on device) == host lookup as values (the sign of zero is not + kept), also for a table of every finite e4m3 code and scales 2^-30..2^30; +* device update (G1 new elementwise/linear ops >= 0.999): the Engram increment ``out - x`` per stream copy and + the output vs the reference's fp32 update; image-token rows pass through bit-exactly; repeat bit-identical; + one and two chunks, and a padded last chunk. + +Real Engram weights and tables (shards 47/48, 101 GB each) are not downloaded: synthetic weights only. +""" + +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41 import engram_reference as er +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config +from models.demos.deepseek_v3_d_p.tt.v41.engram import ( + DEAD, + TtV41Engram, + TtV41EngramTable, + V41EngramHash, + V41EngramTable, +) +from models.demos.deepseek_v3_d_p.tt.v41.weights import dequant_fp8_block +from tests.ttnn.utils_for_testing import comp_pcc + +ENGRAM_PCC = 0.999 +LAYER = 1 + + +def _config(case: str): + return DeepSeekV41FlashConfig if case == "real" else SmallV41Config + + +def _pcc(a, b): + return comp_pcc(a.float(), b.float(), 0.0)[1] + + +def _pack(x, tp): + s, n, d = x.shape + return x.reshape(s, n, tp, d // tp).permute(0, 2, 1, 3).reshape(1, 1, s, n * d) + + +def _unpack(t, n, tp): + s, width = t.shape[-2], t.shape[-1] + return t.reshape(s, tp, n, width // n // tp).permute(0, 2, 1, 3).reshape(s, n, -1) + + +def _hasher(case: str) -> V41EngramHash: + from transformers import AutoTokenizer + + return V41EngramHash.from_tokenizer(_config(case), AutoTokenizer.from_pretrained(orc.HF_SNAPSHOT)) + + +@pytest.mark.parametrize("case", ["small", "small_padded", "small_image", "real"]) +def test_engram_host(case, expect_error): + """Hash ids (chunked, carried history) and row lookup are bit-exact vs the reference.""" + spec, tokens, mask = er.case_inputs(case) + expected = er.engram_case(spec, tokens, mask) + hasher = _hasher(case) + column = hasher.layer_index(LAYER) + total = tokens.numel() + compressed = hasher.token_map[tokens] + if mask is not None: + compressed = torch.where(mask, compressed, DEAD) + splits = {"single": [total], "halves": [256, total - 256], "short": [1, 2, 1, 3, 250, total - 257]} + for name, split in splits.items(): + history, ids, start = hasher.new_history(), [], 0 + for length in split: + chunk_mask = None if mask is None else mask[start : start + length] + chunk_ids, history = hasher(tokens[start : start + length], history, chunk_mask) + ids.append(chunk_ids[:, column]) + start += length + assert torch.equal(history, compressed[max(0, start - 3) : start]), (name, start) + assert torch.equal(torch.cat(ids), expected["hash_ids"]), name + if case == "small": # the helper's Engram input/output are the full oracle's (text-only) captures + block = orc.oracle(spec, tokens[None])["blocks"][LAYER] + assert torch.equal(block["engram_hash_ids"], expected["hash_ids"]) + assert torch.equal(block["engram_in"], expected["x_in"]) and torch.equal(block["x_in"], expected["out"]) + table = V41EngramTable(**expected["table"]) + started = time.perf_counter() + rows = table.lookup(expected["hash_ids"]).flatten(-2) + logger.info(f"{case}: host lookup of {total} tokens {1e3 * (time.perf_counter() - started):.1f} ms") + assert torch.equal(rows, expected["rows"]) + if case == "small": # synthetic rows do not depend on the caller's default dtype + some = expected["table"]["row_ids"][:4] + with v41.set_dtype(torch.bfloat16): + q, sc = orc.synthetic_engram_rows(spec.seed, LAYER, some, 256) + assert torch.equal(q.view(torch.uint8), expected["table"]["weight"][:4].view(torch.uint8)) + assert torch.equal(sc.view(torch.uint8), expected["table"]["scale"][:4].view(torch.uint8)) + with expect_error(KeyError, "does not hold"): + table.lookup(expected["table"]["row_ids"][-1:] + 1) + + +def _table(placement: str, mesh_device, table: V41EngramTable): + return TtV41EngramTable(mesh_device, table) if placement == "device" else table + + +def _code_table(rows: int = 64, seed: int = 0) -> V41EngramTable: + """Every finite e4m3 code in every row (rotated per row), E8M0 scales 2^-30..2^30, sparse global row ids.""" + gen = torch.Generator().manual_seed(seed) + codes = (torch.arange(256)[None] + 37 * torch.arange(rows)[:, None]) % 256 + codes = torch.where((codes & 0x7F) == 0x7F, codes - 1, codes) # 0x7F / 0xFF are NaN + scale = torch.randint(127 - 30, 127 + 31, (rows, 256 // 32), generator=gen) + row_ids = torch.sort(torch.randperm(10**6, generator=gen)[:rows]).values + return V41EngramTable( + codes.to(torch.uint8).view(torch.float8_e4m3fn), + scale.to(torch.uint8).view(torch.float8_e8m0fnu), + row_ids.to(torch.int64), + ) + + +@pytest.mark.timeout(900) +@pytest.mark.parametrize("placement", ["host", "device"]) +@pytest.mark.parametrize("case", ["codes", "small", "real"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) +def test_engram_rows_device(mesh_device, device_params, case, placement): + """Device-dequantized rows are bit-identical to the host lookup, for both table placements.""" + cfg = _config(case) + if case == "codes": + table, chunk = _code_table(), 256 + ids = table.row_ids[ + torch.randint(0, len(table.row_ids), (chunk - 7, 24), generator=torch.Generator().manual_seed(1)) + ] + else: + spec, tokens, mask = er.case_inputs(case) + expected = er.engram_case(spec, tokens, mask) + table, ids, chunk = V41EngramTable(**expected["table"]), expected["hash_ids"], tokens.numel() + hidden, hc = cfg.EMB_SIZE, cfg.HC_MULT + weights = { + "wkv": torch.zeros((hc + 1) * hidden, 24 * 256), + "q_weight": torch.ones(hc, hidden), + "k_weight": torch.ones(hc, hidden), + } + module = TtV41Engram(mesh_device, cfg, LAYER, weights, _table(placement, mesh_device, table)) + shape = tuple(mesh_device.shape) + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + got = down(module.rows(module.prepare(ids, chunk)))[0, 0] + host_ms, device_ms = [], [] # warm repeats: synchronized host wall time of prepare, then of the device rows + for _ in range(3): + began = time.perf_counter() + inputs = module.prepare(ids, chunk) + ttnn.synchronize_device(mesh_device) + host_ms.append(1e3 * (time.perf_counter() - began)) + began = time.perf_counter() + module.rows(inputs) + ttnn.synchronize_device(mesh_device) + device_ms.append(1e3 * (time.perf_counter() - began)) + baseline_ms = [] # the previous host path: dequantize on the host, upload bf16 rows replicated over TP + for _ in range(3 if placement == "host" else 0): + began = time.perf_counter() + rows = torch.zeros(chunk, 24 * 256, dtype=torch.bfloat16) + rows[: ids.shape[0]] = table.lookup(ids).flatten(-2) + ttnn.from_torch( + rows[None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, None)), + ) + ttnn.synchronize_device(mesh_device) + baseline_ms.append(1e3 * (time.perf_counter() - began)) + got = got.view(chunk, shape[1], -1) # each TP replica + want = torch.zeros(chunk, 1, 24 * 256, dtype=torch.bfloat16) + want[: ids.shape[0], 0] = table.lookup(ids).flatten(-2) + want = want.expand_as(got) + # equal as values (-0 == +0: the device does not keep the sign of zero), and no NaN + mismatches = int((got != want).sum()) + signed_zeros = int(((got.view(torch.int16) != want.view(torch.int16)) & (want == 0)).sum()) + logger.info( + f"rows {case}/{placement} (chunk {chunk}): {mismatches} unequal values of {want.numel()} ({signed_zeros} -0 read " + f"as +0); prepare ms {[round(t, 1) for t in host_ms]}, device rows ms {[round(t, 1) for t in device_ms]}" + + (f"; host-dequant bf16 upload ms {[round(t, 1) for t in baseline_ms]}" if baseline_ms else "") + ) + assert mismatches == 0 + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("placement", ["host", "device"]) +@pytest.mark.parametrize("chunks", [1, 2], ids=["single_chunk", "two_chunks"]) +@pytest.mark.parametrize("case", ["small", "small_padded", "small_image", "real"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) +def test_engram_device(mesh_device, device_params, case, chunks, placement): + cfg = _config(case) + spec, tokens, mask = er.case_inputs(case) + expected = er.engram_case(spec, tokens, mask) + reference = orc.build_reference(spec) + engram = reference.layers[0].engram + weights = { + "wkv": dequant_fp8_block(engram.wkv.weight.detach(), engram.wkv.scale.detach()), + "q_weight": engram.q_weight.detach(), + "k_weight": engram.k_weight.detach(), + } + hasher = V41EngramHash(cfg, reference.engram_hash.token_map) + column = hasher.layer_index(LAYER) + shape, (sp, tp), n = tuple(mesh_device.shape), tuple(mesh_device.shape), cfg.HC_MULT + table = _table(placement, mesh_device, V41EngramTable(**expected["table"])) + module = TtV41Engram(mesh_device, cfg, LAYER, weights, table) + del reference, weights + + total = tokens.numel() + chunk = -(-total // chunks // 128) * 128 # a multiple of 2 * 32 * sp + x_in = expected["x_in"].float() + + def run(): + outs, history, host_ms, device_ms = [], hasher.new_history(), [], [] + for start in range(0, total, chunk): + length = min(chunk, total - start) + chunk_mask = None if mask is None else mask[start : start + length] + began = time.perf_counter() + ids, history = hasher(tokens[start : start + length], history, chunk_mask) + inputs = module.prepare(ids[:, column], chunk, chunk_mask) + ttnn.synchronize_device(mesh_device) + host_ms.append(1e3 * (time.perf_counter() - began)) + rows = torch.zeros(chunk, n, x_in.shape[-1]) + rows[:length] = x_in[start : start + length] + x = ttnn.from_torch( + _pack(rows, tp), + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + ttnn.synchronize_device(mesh_device) + began = time.perf_counter() + out = module(x, inputs) + ttnn.synchronize_device(mesh_device) + device_ms.append(1e3 * (time.perf_counter() - began)) + full = ttnn.to_torch(out, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + outs.append(_unpack(full[0, 0], n, tp)[:length]) + return torch.cat(outs), host_ms, device_ms + + first, host_ms, device_ms = run() + second, host_ms2, device_ms2 = run() + want = expected["out_fp32"] + delta, want_delta = first - x_in, want - x_in + report = { + "out": _pcc(want, first), + "out_vs_bf16_reference": _pcc(expected["out"], first), + "delta_per_copy": [_pcc(want_delta[:, c], delta[:, c]) for c in range(n)], + "delta_max_abs_err": float((delta - want_delta).abs().max()), + "deterministic": torch.equal(first, second), + "host_prepare_ms": [round(t, 1) for t in host_ms2], + "device_forward_ms": [round(t, 1) for t in device_ms2], + } + if mask is not None: + report["image_rows_unchanged"] = torch.equal(first[~mask], x_in[~mask]) + logger.info(f"engram {case} {chunks} chunk(s), {placement} table: {report}") + assert report["deterministic"] + assert report.get("image_rows_unchanged", True) + assert report["out"] >= ENGRAM_PCC and min(report["delta_per_copy"]) >= ENGRAM_PCC, report diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_head_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_head_v41.py new file mode 100644 index 000000000000..4f70b1ff1173 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_head_v41.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 embedding and output head (bead F1) vs the reference ``ParallelEmbedding``, final ``norm`` +and ``ParallelHead`` at V4.1 dims, synthetic and checkpoint weights (embed: shard 02, norm/head: shard 43). + +Inputs come from the §6 oracle of V4.1 layer 2 on one 5120-token chunk (the same run as +``test_moe_v41``): its prompt tokens for the embedding, and the collapsed final hidden +``hc_pre(x_out, pre_out)`` for the head. The head is evaluated on the last row (the reference's own +logits) and on rows spread over SP chips and tile offsets. + +Bars: embedding exact (``torch.equal``); head logits PCC >= 0.999 (new linear op bar) and identical top-1 +on every row; bit-identical repeats. +""" + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as O +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.v41.head import TtV41Embedding, TtV41Head +from tests.ttnn.utils_for_testing import comp_pcc + +LAYER = 2 +SEQ = 5120 +HEAD_PCC = 0.999 +ROWS = (SEQ - 1, 0, 31, 32, 1234, 1279, 1280, 2559, 2560, 3999) + +MESHES = [ + pytest.param( + shape, + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=shape, topology=f"mesh-{shape[0]}x{shape[1]}"), + id=f"fabric2d-mesh-{shape[0]}x{shape[1]}", + ) + for shape in ((2, 4), (4, 2)) +] + + +def _reference(source: str): + if source == "real" and not (O.HF_SNAPSHOT / "model.safetensors.index.json").is_file(): + pytest.skip("V4.1 checkpoint shards not downloaded") + spec = O.real_spec((LAYER,), SEQ, checkpoint=O.HF_SNAPSHOT if source == "real" else None) + model = O.build_reference(spec) + result = O.oracle(spec, O.random_tokens(spec), model) + return model, result + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("weights_source", ["synthetic", "real"]) +@pytest.mark.parametrize("mesh_device, device_params", MESHES + galaxy_meshes(), indirect=True) +def test_v41_embedding_head(mesh_device, device_params, weights_source): + model, result = _reference(weights_source) + shape = tuple(mesh_device.shape) + concat = ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3)) + + # --- embedding: exact lookup + tokens = result["tokens"] + with torch.no_grad(): + expected_embed = model.embed(tokens[None].clone())[0] + embed = TtV41Embedding(mesh_device, C, model.embed.weight.data) + tt_ids = ttnn.from_torch( + tokens.to(torch.int32).reshape(1, 1, SEQ), + device=mesh_device, + dtype=ttnn.uint32, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, None)), + ) + embeds = [ttnn.to_torch(embed(tt_ids), mesh_composer=concat).reshape(SEQ, C.EMB_SIZE) for _ in range(2)] + assert torch.equal(embeds[0], embeds[1]), "embedding is not bit-identical across repeats" + assert torch.equal(embeds[0], expected_embed), "embedding differs from the reference lookup" + del embed + + # --- final norm + head on single rows + block = result["blocks"][LAYER] + with torch.no_grad(), v41.set_dtype(torch.bfloat16): + hidden = model.layers[0].hc_pre(block["x_out"][None], block["pre_out"][None]) # [1, S, D] bf16 + rows = list(ROWS) + # one row at a time, shaped exactly as the reference head sees its last position + expected = torch.stack([model.head(model.norm(hidden[:, r : r + 1]))[0] for r in rows]) # fp32 + assert torch.equal(expected[0], result["logits"]), "reference head differs from the oracle's last-position logits" + + head = TtV41Head(mesh_device, C, model.norm.weight.data, model.head.weight.data.to(torch.bfloat16)) + tt_hidden = ttnn.from_torch( + hidden[None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + report = {} + for i, row in enumerate(rows): + runs = [head.logits_to_host(*head(tt_hidden, row)) for _ in range(2 if i == 0 else 1)] + if i == 0: + assert torch.equal(runs[0], runs[1]), "head logits are not bit-identical across repeats" + got, want = runs[0], expected[i] + assert torch.isfinite(got).all(), f"row {row}: non-finite logits" + report[row] = { + "pcc": comp_pcc(want, got, 0.0)[1], + "top1": (int(want.argmax()), int(got.argmax())), + "max_abs_err": (want - got).abs().max().item(), + } + logger.info(f"V41_HEAD_RESULT mesh={list(shape)} weights={weights_source} {report}") + for row, r in report.items(): + assert r["pcc"] >= HEAD_PCC, (row, r) + assert r["top1"][0] == r["top1"][1], (row, r) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_indexer.py b/models/demos/deepseek_v3_d_p/tests/v41/test_indexer.py new file mode 100644 index 000000000000..9c1a08225774 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_indexer.py @@ -0,0 +1,1441 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 index keys, indexer, candidates, top-k (beads F4, F10) vs the reference at real dims, and the +selection kernels (``tt/v41/indexer_kernels.py``) vs torch. + +``test_v41_indexer`` uses the oracle's V4.1 layers 2 -> 3 -> 20 -> 21 -> 24 reference (synthetic weights, 96 +candidate blocks) and tests layers 2 (ratio-2 KV+index source), 20 (ratio-1 candidate source), 24 +(candidate-constrained index source). Bars (G1): index scores >= 0.999; selection exact given identical scores +(checked on tie-free scores fed to both sides, through the module's ``select``: candidate block sets and top-k +row +sets, with 96 candidate blocks (one-level candidates) and 32 (two-level)); determinism. Recall of the device +selection against the reference under device scores is reported as a diagnostic only. + +``test_v41_indexer_selection_long`` feeds tie-free scores of chunks starting at 0 / 20,480 / 65,536 with the +production 2048 candidate blocks (all blocks candidates / one-level / two-level candidates and the gathered +candidate top-k, ``segment_max`` / ``gather_runs`` underneath) and checks the same exactness against +``v41.select_candidate_blocks`` + ``torch.topk``. +""" + +import time + +import pytest +import torch + +import ttnn +from models.common.timing_events import phase +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.kernel_cpu import fp4_act_quant +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.reference_weights import dequant +from models.demos.deepseek_v3_d_p.tt.v41.cache import V41ChunkTables +from models.demos.deepseek_v3_d_p.tt.v41.indexer import ( + SENTINEL, + CandidateBlocks, + DecodePins, + TtV41Indexer, + TtV41IndexKeys, +) +from models.demos.deepseek_v3_d_p.tt.v41.indexer_kernels import ( + RANGE_CHUNK, + candidate_scores, + gather_runs, + range_scores, + segment_max, + split_geometry, +) +from tests.ttnn.utils_for_testing import comp_pcc + +SCORE_PCC = 0.999 +LAYERS = (2, 3, 20, 21, 24) +SEQ = 2048 +MESH_2X4 = [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) +] +# the kernel tests derive every shape from the mesh: they run on the Galaxy layouts too (the 2x4 cases skip +# there) +KERNEL_MESHES = MESH_2X4 + galaxy_meshes() + + +def _per_query(t): + """Per-query device outputs (queries split over SP then TP) -> host [1, 1, S, W] in token order.""" + return torch.cat([ttnn.to_torch(d) for d in ttnn.get_device_tensors(t)], dim=2) + + +def _feed(mesh_device, scores: torch.Tensor): + """Host [S, W] scores -> per-chip row-major [1, 1, S/(sp*tp), W] bf16 (the indexer's score layout).""" + sp, tp = tuple(mesh_device.shape) + seq, width = scores.shape + return ttnn.from_torch( + scores.to(torch.bfloat16).reshape(sp, tp, seq // (sp * tp), width), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, 1)), + ) + + +def _rows(t: torch.Tensor) -> list[set]: + """[S, k] device picks (sentinel = none) -> per-query sets; asserts the sentinels form a tail.""" + t = t.long() + out = [] + for row in t: + valid = row != SENTINEL + n = int(valid.sum()) + assert bool(valid[:n].all()), "sentinel picks are not a tail" + out.append(set(row[:n].tolist())) + return out + + +def _ref_rows(scores: torch.Tensor, k: int) -> list[set]: + top = scores.topk(k, dim=-1) + return [set(i[v > float("-inf")].tolist()) for i, v in zip(top.indices, top.values)] + + +def _ref_blocks(mask: torch.Tensor, block: int) -> list[set]: + """``select_candidate_blocks`` row mask [S, W] -> per-query kept block sets.""" + return [set((torch.nonzero(row[::block]).flatten()).tolist()) for row in mask] + + +def _check_selection(mesh_device, cfg, tables, scores: torch.Tensor, start: int, results: dict, tag: str): + """Exactness of the candidate source's blocks and top-k and of a candidate index source's top-k on the + same + host ``scores`` [S, W] (-inf where not visible), through ``TtV41Indexer.select``.""" + seq, width = scores.shape + visible = start + seq + weights = { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + } + source = TtV41Indexer(mesh_device, cfg, cfg.CANDIDATE_SOURCE_LAYER, weights) + consumer = TtV41Indexer(mesh_device, cfg, 24, weights) + feed = _feed(mesh_device, scores) + k = min(cfg.INDEX_TOPK, visible) + compress_lens = (torch.arange(start + 1, start + seq + 1)).unsqueeze(-1) + ref_mask = v41.select_candidate_blocks( + scores.clone(), compress_lens, cfg.CANDIDATE_TOPK_BLOCKS, cfg.CANDIDATE_BLOCK_SIZE + ) + idx, published = source.select(feed, tables, start, visible) + assert isinstance(published, CandidateBlocks) + if published.ids is None: + results[f"{tag}_candidates_exact"] = bool(ref_mask[torch.isfinite(scores)].all()) + else: + dev_blocks = _rows(_per_query(published.ids)[0, 0]) + results[f"{tag}_candidates_exact"] = dev_blocks == _ref_blocks(ref_mask, cfg.CANDIDATE_BLOCK_SIZE) + ref_rows = _ref_rows(scores, k) + results[f"{tag}_source_topk_exact"] = _rows(_per_query(idx)[0, 0]) == ref_rows + if published.ids is not None and cfg.CANDIDATE_TOPK_BLOCKS > k: + # the source's top-k among its own candidate blocks (its path on rows wider than + # SUBSET_TOPK_MIN_WIDTH) + idx = source._topk_in_blocks(feed, published.ids, k) + results[f"{tag}_source_topk_in_blocks_exact"] = _rows(_per_query(idx)[0, 0]) == ref_rows + idx, _ = consumer.select(feed, tables, start, visible, published) + masked = scores.masked_fill(~ref_mask, float("-inf")) + ref_masked = _ref_rows(masked, k) + results[f"{tag}_consumer_topk_exact"] = _rows(_per_query(idx)[0, 0]) == ref_masked + return published + + +def _reference_scores(attn, x, qr): + """The reference Indexer.forward score for a single-shot prefill (start 0), before candidate masking.""" + ind = attn.indexer + seq, ratio, rd = x.shape[1], ind.compress_ratio, ind.rope_head_dim + q = ind.wq_b(qr).unflatten(-1, (ind.n_local_heads, ind.index_head_dim)) + v41.apply_rotary_emb(q[..., -rd:], ind.freqs_cis[:seq]) + fp4_act_quant(q, 32, True) + index_k = v41.shared_attn.index_k[:1, : seq // ratio] + weights = ind.weights_proj(x) * (ind.softmax_scale * ind.n_heads**-0.5) + score = torch.einsum("bshd,btd->bsht", q, index_k) + score = (score.relu_() * weights.unsqueeze(-1)).sum(dim=2) + compress_lens = (torch.arange(1, seq + 1) // ratio).unsqueeze(-1) + score.masked_fill_(torch.arange(seq // ratio) >= compress_lens, -torch.inf) + return score[0].float() + + +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("mesh_device, device_params", MESH_2X4, indirect=True) +def test_v41_indexer(mesh_device, device_params): + cfg = type("V41TestConfig", (C,), {"CANDIDATE_TOPK_BLOCKS": 96}) # match the oracle's candidate count + spec = orc.real_spec(LAYERS, SEQ, candidate_topk_blocks=96) + reference = orc.build_reference(spec) + result = orc.oracle(spec, orc.random_tokens(spec), model=reference) + shape, (sp, tp) = tuple(mesh_device.shape), tuple(mesh_device.shape) + seq = SEQ + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + + per_query = _per_query + + def up(t, dims=(2, None), dtype=ttnn.bfloat16): + return ttnn.from_torch( + t, + device=mesh_device, + dtype=dtype, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=dims), + ) + + results, real_candidates = {}, None + tables = V41ChunkTables(mesh_device, cfg, seq, seq, list(LAYERS)) # the forward's position tables + for layer in (2, 20, 24): + attn = reference.layers[LAYERS.index(layer)].attn + x = result["blocks"][layer]["attn_in"][None] + with v41.set_dtype(torch.bfloat16): + qr = attn.q_norm(attn.wq_a(x)) + _, ref_idx = attn._compress_kv(x, qr, 0, seq) # runs the reference compressor / keys / indexer + ref_idx = ref_idx[0].long() - seq # [S, k] rows, -1 - seq for none + ratio = cfg.compress_ratio(layer) + ind_w = {"wq_b": dequant(attn.indexer.wq_b), "weights_proj": attn.indexer.weights_proj.weight.detach()} + indexer = TtV41Indexer(mesh_device, cfg, layer, ind_w) + + # index keys (KV sources) from the reference latent -> compare with the reference's index-K rows + src = cfg.kv_source(layer) + ref_k = v41.shared_attn.index_k[0, : seq // ratio].float() + if layer in cfg.KV_SOURCE_LAYERS: + keys = TtV41IndexKeys( + mesh_device, + cfg, + layer, + {"wk": attn.indexer.wk.weight.detach(), "k_norm": attn.indexer.k_norm.weight.detach()}, + ) + with v41.set_dtype(torch.bfloat16): + latent = attn.compressor(x, 0)[0] + dev_k = down(keys(up(latent[None, None]), tables.rope(True, ratio, 0)))[0, 0, :, : cfg.INDEX_HEAD_DIM] + results[f"L{layer}_index_keys"] = comp_pcc(ref_k, dev_k.float(), 0.0)[1] + index_k = ttnn.from_torch( + ref_k.to(torch.bfloat16)[None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + tt_x, tt_qr = up(x[0][None, None], dims=(2, 3)), up(qr[0][None, None]) + + score, visible = indexer.scores(tt_x, tt_qr, index_k, tables, 0, seq) + dev_score = per_query(score)[0, 0, :, :visible].float() + with v41.set_dtype(torch.bfloat16): + ref_score = _reference_scores(attn, x, qr) + finite = torch.isfinite(ref_score) + assert torch.equal(finite, torch.isfinite(dev_score)), f"L{layer}: visibility masks differ" + results[f"L{layer}_scores"] = comp_pcc(ref_score[finite], dev_score[finite], 0.0)[1] + + if layer > cfg.CANDIDATE_SOURCE_LAYER and real_candidates.ids is not None: + # candidate scoring (the long-row path of a candidate index source) vs the reference and the + # full-row + # score, both over the candidate rows + ids = real_candidates.ids + sparse = per_query(indexer.candidate_rows(tt_x, tt_qr, index_k, tables, 0, ids))[0, 0].float() + dense = per_query(gather_runs(score, ids, cfg.CANDIDATE_BLOCK_SIZE))[0, 0].float() + host_ids = per_query(ids)[0, 0].long() & 0xFFFFFFFF + cols = (host_ids.unsqueeze(-1) * cfg.CANDIDATE_BLOCK_SIZE + torch.arange(cfg.CANDIDATE_BLOCK_SIZE)).flatten( + 1 + ) + padded = torch.cat([ref_score, torch.full((seq, 1), float("-inf"))], dim=1) + ref_rows = torch.gather(padded, 1, cols.clamp(max=ref_score.shape[1])) + cand_finite = torch.isfinite(ref_rows) + results[f"L{layer}_candidate_rows_mask_exact"] = torch.equal(torch.isfinite(sparse), cand_finite) and ( + torch.equal(torch.isfinite(dense), cand_finite) + ) + results[f"L{layer}_candidate_scores"] = comp_pcc(ref_rows[cand_finite], sparse[cand_finite], 0.0)[1] + results[f"L{layer}_full_row_candidate_scores_diag"] = comp_pcc( + ref_rows[cand_finite], dense[cand_finite], 0.0 + )[1] + + # selection given identical, tie-free scores + g = torch.Generator().manual_seed(layer) + # distinct bf16 values per row: distinct positive bf16 bit patterns (integers > 256 would collide) + codes = torch.stack([torch.randperm(0x4000, generator=g)[:visible] + 0x3000 for _ in range(seq)]) + rows = codes.to(torch.int16).view(torch.bfloat16).float() + tie_free = torch.where(finite, rows, float("-inf")) + if layer == cfg.CANDIDATE_SOURCE_LAYER: + # candidate source + candidate index source on the same scores: one-level (96 blocks of 256 per + # row, 64 superblocks) and two-level (32 blocks) candidate selection + for blocks in (96, 32): + sel_cfg = type(f"V41TestConfigK{blocks}", (C,), {"CANDIDATE_TOPK_BLOCKS": blocks}) + _check_selection(mesh_device, sel_cfg, tables, tie_free, 0, results, f"K{blocks}") + real_candidates = indexer.candidates(score, tables, 0) + elif layer < cfg.CANDIDATE_SOURCE_LAYER: + idx, _ = indexer.select(_feed(mesh_device, tie_free), tables, 0, visible) + k = min(cfg.INDEX_TOPK, visible) + results[f"L{layer}_selection_exact"] = _rows(per_query(idx)[0, 0]) == _ref_rows(tie_free, k) + + # determinism of the full indexer on real scores + cands = real_candidates if layer > cfg.CANDIDATE_SOURCE_LAYER else None + idx1, _ = indexer(tt_x, tt_qr, index_k, tables, 0, seq, cands) + idx2, _ = indexer(tt_x, tt_qr, index_k, tables, 0, seq, cands) + a, b = per_query(idx1)[0, 0], per_query(idx2)[0, 0] + results[f"L{layer}_deterministic"] = bool(torch.equal(a, b)) + k = min(cfg.INDEX_TOPK, visible) + dev_rows = torch.where(a[:, :k].long() == SENTINEL, -1, a[:, :k].long()) + ref_rows = torch.where(ref_idx >= 0, ref_idx, -1) + recall = [ + len(set(r[r >= 0].tolist()) & set(d[d >= 0].tolist())) / max(1, int((r >= 0).sum())) + for r, d in zip(ref_rows, dev_rows) + ] + results[f"L{layer}_recall_diag"] = sum(recall) / len(recall) + print(f"indexer: {results}") + for key, value in results.items(): + if key.endswith("_scores") or key.endswith("_index_keys"): + assert value >= SCORE_PCC, (key, results) + if key.endswith("_exact") or key.endswith("_deterministic"): + assert value, (key, results) + + +def _tie_free_scores(seq: int, start: int, g) -> torch.Tensor: + """[seq, W] scores of a ratio-1 chunk at ``start`` (-inf where not visible, W = visible rows rounded to + 32): + per row a random 30 % (at least 4096) of the visible rows get distinct positive bf16 values and the rest + one lower + value, so block maxima and the top picks are tie-free while rows may hold more elements than bf16 has + distinct + values.""" + width = _round32(start + seq) + p = torch.arange(start, start + seq).view(-1, 1) + t = torch.arange(width).view(1, -1) + out = torch.full((seq, width), -100.0) + for i in range(seq): + n = start + i + 1 + top = torch.randperm(n, generator=g)[: min(n, max(4096, (3 * n) // 10))] + codes = torch.randperm(0x7F00 - 0x0080, generator=g)[: top.numel()] + 0x0080 + out[i, top] = codes.to(torch.int16).view(torch.bfloat16).float() + return torch.where(t <= p, out, float("-inf")) + + +def _round32(x: int) -> int: + return -(-x // 32) * 32 + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("start", [0, 20480, 65536], ids=lambda s: f"start{s}") +@pytest.mark.parametrize("mesh_device, device_params", MESH_2X4, indirect=True) +def test_v41_indexer_selection_long(mesh_device, device_params, start): + """Production candidate count (2048 blocks) on long rows: all-candidates / one-level / two-level paths.""" + seq = 1024 + g = torch.Generator().manual_seed(start) + with phase("reference", what="tie-free scores"): + scores = _tie_free_scores(seq, start, g) + results = {} + with phase("compute", what="tables + select + reference selection"): + tables = V41ChunkTables(mesh_device, C, start + seq, seq, [2, 20, 24]) + _check_selection(mesh_device, C, tables, scores, start, results, f"start{start}") + print(f"indexer selection: {results}") + assert all(results.values()), results + + +def _decode_scores(rows: int, cap: int, visible: torch.Tensor, g) -> torch.Tensor: + """[rows, cap + 32] scores of a decode tile over a pool of ``cap`` rows (the pad tile after them): row i + sees + its first ``visible[i]`` rows, -inf beyond; an idle row (0) is all -inf. Tie-free where it matters: up to + 4096 + of a row's blocks hold one distinct positive bf16 code each (at a random row of the block), the rest of + the + visible rows are -100, so the top-2048 blocks by maximum, the top-k rows and the top-k inside the + candidate + blocks are unambiguous. The codes stay below 2^32: the segment max's -inf column masks saturate against + values + above ~1e36 on the FPU (`-inf + x` -> `-bf16max + x`), which real scores never reach.""" + width = cap + 32 + out = torch.full((rows, width), float("-inf")) + for i in range(rows): + n = int(visible[i]) + if n == 0: + continue + out[i, :n] = -100.0 + nblocks = -(-n // 8) + blocks = torch.randperm(nblocks, generator=g)[: min(nblocks, 4096)] + pos = (blocks * 8 + torch.randint(0, 8, (blocks.numel(),), generator=g)).clamp(max=n - 1) + # and enough distinct positives for a tie-free top-k (INDEX_TOPK) wherever the row has that many rows + pos = torch.unique(torch.cat([pos, torch.randperm(n, generator=g)[: min(n, 8192)]])) + codes = torch.randperm(0x4F00 - 0x0080, generator=g)[: pos.numel()] + 0x0080 + out[i, pos] = codes.to(torch.int16).view(torch.bfloat16).float() + return out + + +def _replicated(mesh_device, host: torch.Tensor, dtype): + return ttnn.from_torch( + host, + device=mesh_device, + dtype=dtype, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + + +def _chip0(t) -> torch.Tensor: + return ttnn.to_torch(ttnn.get_device_tensors(t)[0])[0, 0] + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("cap", [8192, 16384, 20480, 69632, 1048576], ids=lambda c: f"cap{c}") +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_decode_selection(mesh_device, device_params, cap): + """The decode tile's selection (``TtV41Indexer.select`` with ``DecodePins``) against the reference: 32 + rows at + their own visible counts (idle rows, a row below INDEX_TOPK, block and superblock boundaries, the full + pool), + the candidate source's published blocks vs ``select_candidate_blocks`` with that row's count, its top-k vs + the + dense top-k, a consumer's top-k vs its dense score masked by the published blocks. Capacities on the + all-candidates, one-level and two-level paths (the last two below and above SUBSET_TOPK_MIN_WIDTH).""" + rows, cfg = 32, C + g = torch.Generator().manual_seed(cap) + visible = torch.randint(1, cap + 1, (rows,), generator=g) + visible[0], visible[1] = 0, 0 # idle users: the sentinel pin, an all -inf row + visible[2] = 300 # fewer visible rows than INDEX_TOPK: a sentinel tail + visible[3] = cap # the whole pool + visible[4] = 8 * (cap // 16) # a block boundary (the newest block is full) + visible[5] = 8 * (cap // 16) + 1 # the row after it (a one-row newest block) + visible[6] = 32 * (cap // 64) + 5 # inside a superblock + visible[7] = min(cap, 2048 * 8 + 3) # just past the budget of one level + with phase("reference", what="tie-free decode scores + reference selection"): + scores = _decode_scores(rows, cap, visible, g) + k = cfg.INDEX_TOPK + ref_blocks, ref_src, ref_cons = [], [], [] + for i in range(rows): + n = int(visible[i]) + if n == 0: + ref_blocks.append(None), ref_src.append(None), ref_cons.append(None) + continue + mask = v41.select_candidate_blocks( + scores[i : i + 1].clone(), n, cfg.CANDIDATE_TOPK_BLOCKS, cfg.CANDIDATE_BLOCK_SIZE + ) + ref_blocks.append(_ref_blocks(mask, cfg.CANDIDATE_BLOCK_SIZE)[0]) + ref_src.append(_ref_rows(scores[i : i + 1], k)[0]) + ref_cons.append(_ref_rows(scores[i : i + 1].masked_fill(~mask, float("-inf")), k)[0]) + with phase("compute", what="device selection"): + weights = { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + } + source = TtV41Indexer(mesh_device, cfg, cfg.CANDIDATE_SOURCE_LAYER, weights) + consumer = TtV41Indexer(mesh_device, cfg, 24, weights) + feed = _replicated(mesh_device, scores.to(torch.bfloat16)[None, None], ttnn.bfloat16) + newest = torch.where(visible > 0, visible - 1, torch.full_like(visible, -1)).to(torch.int32) + pins = DecodePins(_replicated(mesh_device, newest.view(1, 1, 1, rows), ttnn.uint32)) + idx, published = source.select(feed, pins, 0, k) + assert isinstance(published, CandidateBlocks) + src_rows = _rows(_chip0(idx)) + dev_blocks = None if published.ids is None else _rows(_chip0(published.ids)) + idx, _ = consumer.select(feed, pins, 0, k, published) + cons_rows = _rows(_chip0(idx)) + results = {} + for i in range(rows): + if ref_src[i] is None: + continue + if dev_blocks is None: + results[f"row{i}_blocks"] = cap // cfg.CANDIDATE_BLOCK_SIZE <= cfg.CANDIDATE_TOPK_BLOCKS + else: + results[f"row{i}_blocks"] = dev_blocks[i] == ref_blocks[i] + results[f"row{i}_source"] = src_rows[i] == ref_src[i] + results[f"row{i}_consumer"] = cons_rows[i] == ref_cons[i] + bad = {key: value for key, value in results.items() if not value} + print(f"decode selection cap {cap}: {len(results)} checks, {len(bad)} wrong: {sorted(bad)[:12]}") + assert not bad, sorted(bad) + + +def _pin_table(mesh_device, rows: int): + """Per-chip uint32 [1, 1, 1, 8] first chunk query index, as ``V41ChunkTables.query_first``.""" + sp, tp = tuple(mesh_device.shape) + first = torch.zeros(sp, tp, 1, 8, dtype=torch.int32) + first[..., 0] = (torch.arange(sp * tp, dtype=torch.int32) * rows).reshape(sp, tp, 1) + return ttnn.from_torch( + first, + device=mesh_device, + dtype=ttnn.uint32, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, 1)), + ) + + +def _pin_rows(mesh_device, newest: torch.Tensor, rows: int): + """Per-chip uint32 [1, 1, 1, rows] per-row pin page: entry i the chip's row i's value of ``newest`` [seq].""" + sp, tp = tuple(mesh_device.shape) + return ttnn.from_torch( + newest.to(torch.int64).reshape(sp, tp, 1, rows).to(torch.int32), # the sentinel wraps to -1: its bits + device=mesh_device, + dtype=ttnn.uint32, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, 1)), + ) + + +def _kernel_rows(seq: int, width: int, g) -> torch.Tensor: + """[seq, width] bf16 rows with negatives, zeros, repeated values and -inf runs (whole blocks, partial + blocks and + a -inf row tail).""" + x = (torch.randn(seq, width, generator=g) * 4).to(torch.bfloat16).float() + x[:, ::7] = -x[:, ::7].abs() - 1 # plenty of all-negative blocks + x[::5, 3:40] = float("-inf") + x[1::3, width // 2 :] = float("-inf") + x[2, :] = -3.0 # a row of ties + x[3, 10:20] = 0.0 + return x.to(torch.bfloat16).float() + + +@pytest.mark.parametrize("width", [1056, 8288, 16384], ids=lambda w: f"W{w}") +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_segment_max(mesh_device, device_params, width): + """segment_max vs torch (exact, every chip): seg 8 / 32, partial last chunk and padded output widths, + negative / -inf / tied values, pins inside, at the edge of and past the row, and the rank-0 (masked) + pin.""" + sp, tp = tuple(mesh_device.shape) + rows = 64 # per chip + seq = rows * sp * tp + g = torch.Generator().manual_seed(width) + x = _kernel_rows(seq, width, g) + feed = _feed(mesh_device, x) + pin = _pin_table(mesh_device, rows) + q = torch.arange(seq) + for seg in (8, 32): + out_width = -(-(width // seg) // 32) * 32 + base = x.reshape(seq, width // seg, seg).amax(dim=-1) + for pin_base, pin_mask in ((width - seq // 2, 0xFFFFFFFF), (0, 31), (width, 0xFFFFFFFF)): + ref = torch.full((seq, out_width), float("-inf")) + ref[:, : width // seg] = base + e = (pin_base + q) & pin_mask + hit = e < width + ref[q[hit], e[hit] // seg] = float("inf") + out = segment_max(feed, seg, pin, pin_base, pin_mask) + assert list(out.shape) == [1, 1, rows, out_width] + dev = _per_query(out)[0, 0].float() + assert torch.equal(dev, ref), (seg, pin_base, pin_mask, (dev != ref).nonzero()[:8].tolist()) + again = _per_query(segment_max(feed, seg, pin, pin_base, pin_mask))[0, 0].float() + assert torch.equal(again, dev), "segment_max is not deterministic" + # the per-row pin page (the decode tile): row i pinned at its own entry, sentinel rows not pinned + newest = torch.randint(0, width + 64, (seq,)) # some past the row: not pinned + newest[1::7] = SENTINEL + for pin_base, pin_mask in ((0, 0xFFFFFFFF), (0, 31)): + ref = torch.full((seq, out_width), float("-inf")) + ref[:, : width // seg] = base + e = (pin_base + newest) & pin_mask + hit = (e < width) & (newest != SENTINEL) + ref[q[hit], e[hit] // seg] = float("inf") + dev = _per_query(segment_max(feed, seg, _pin_rows(mesh_device, newest, rows), pin_base, pin_mask)) + dev = dev[0, 0].float() + assert torch.equal(dev, ref), ("per-row", seg, pin_mask, (dev != ref).nonzero()[:8].tolist()) + + +@pytest.mark.parametrize("k", [96, 2048], ids=lambda k: f"K{k}") +@pytest.mark.parametrize("mesh_device, device_params", MESH_2X4, indirect=True) +def test_v41_gather_runs(mesh_device, device_params, k): + """gather_runs vs torch (exact, every chip): runs of 8 and 32, ids in any order with repeats, the sentinel + and + ids just past the row reading -inf.""" + sp, tp = tuple(mesh_device.shape) + rows, width = 64, 65536 + 96 + seq = rows * sp * tp + g = torch.Generator().manual_seed(k) + x = _kernel_rows(seq, width, g) + feed = _feed(mesh_device, x) + for run in (8, 32): + nruns = width // run + ids = torch.randint(0, nruns, (seq, k), generator=g) + ids[:, -3] = nruns # first id past the row + ids[::2, -1] = SENTINEL + ids = ids & 0xFFFFFFFF + ids[:, 0] = nruns - 1 # the row's last run + ids[:, 1] = ids[:, 2] # a repeat + dev_ids = ttnn.from_torch( + ids.to(torch.int32).reshape(sp, tp, rows, k), # the sentinel wraps to -1: its uint32 bits + device=mesh_device, + dtype=ttnn.uint32, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, 1)), + ) + cols = ids.clamp(max=nruns).unsqueeze(-1) * run + torch.arange(run) # [seq, k, run] + padded = torch.cat([x, torch.full((seq, run), float("-inf"))], dim=1) + ref = torch.gather(padded, 1, cols.clamp(max=width).reshape(seq, -1)) + out = gather_runs(feed, dev_ids, run) + assert list(out.shape) == [1, 1, rows, k * run] + dev = _per_query(out)[0, 0].float() + assert torch.equal(dev, ref), (run, (dev != ref).nonzero()[:8].tolist()) + assert torch.equal(_per_query(gather_runs(feed, dev_ids, run))[0, 0].float(), dev), "not deterministic" + + +def _fp4_like(shape, g) -> torch.Tensor: + """bf16 values with at most 2 significant bits (FP4 e2m1 codes times a power of two), as QDQ'd q and + index-K.""" + codes = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) + v = codes[torch.randint(0, 8, shape, generator=g)] * torch.where(torch.rand(shape, generator=g) < 0.5, -1.0, 1.0) + return (v * 2.0 ** torch.randint(-4, 0, shape[:-1] + (1,), generator=g)).to(torch.bfloat16) + + +def _candidate_score_ref(q, w, k, ids, positions) -> torch.Tensor: + """[S, K * 8] reference of ``candidate_scores``: per head a bf16 q . k (exact fp32 dot, one rounding), + relu, + weighted sum over the heads in fp32, one bf16 rounding; -inf for sentinel ids and rows past the query. + Also + returns sum_h |w_h relu(q_h . k)| (the scale of the head sum's accumulation error).""" + seq, nk = ids.shape + nblocks = k.shape[0] // 8 + valid = ids < nblocks + rows = (ids.clamp(max=nblocks - 1).unsqueeze(-1) * 8 + torch.arange(8)).reshape(seq, -1) # [S, K * 8] + out, magnitude = torch.empty(seq, nk * 8), torch.empty(seq, nk * 8) + for i in range(seq): + kr = k[rows[i]].float() # [K * 8, 128] + s = torch.einsum("hd,td->ht", q[i].float(), kr).to(torch.bfloat16).float().relu() + terms = w[i].float().unsqueeze(-1) * s + out[i] = terms.sum(0).to(torch.bfloat16).float() + magnitude[i] = terms.abs().sum(0) + visible = valid.repeat_interleave(8, dim=1) & (rows <= positions.view(-1, 1)) + return torch.where(visible, out, float("-inf")), magnitude + + +@pytest.mark.parametrize("nk, start", [(96, 2048), (2048, 131072)], ids=["K96", "K2048"]) +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_candidate_scores(mesh_device, device_params, nk, start): + """candidate_scores vs torch (every chip): the newest block pinned first (partly past the query), random + blocks + with a repeat, a sentinel tail on some queries; FP4-like q / index-K, bf16 head weights. The finite + pattern must + match exactly; values within one bf16 rounding plus 2^-6 of the head terms' magnitude of the + fp32-accumulated + reference, PCC >= 0.9999; deterministic.""" + sp, tp = tuple(mesh_device.shape) + rows = 32 # per chip + seq = rows * sp * tp + g = torch.Generator().manual_seed(nk) + with phase("reference", what="inputs + torch candidate scores"): + t_rows = start + seq + q = _fp4_like((seq, 32, 128), g) + k = _fp4_like((t_rows, 128), g) + w = (torch.randn(seq, 32, generator=g) * 0.05).to(torch.bfloat16) + positions = start + torch.arange(seq) + ids = torch.stack([torch.randperm(start // 8, generator=g)[:nk] for _ in range(seq)]) + ids[:, 0] = positions // 8 # the pinned newest block: rows past the query are masked + ids[:, 2] = ids[:, 3] # a repeat + ids[::3, nk - 5 :] = SENTINEL + ref, magnitude = _candidate_score_ref(q, w, k, ids, positions) + with phase("compute", what="device candidate scores"): + shard = ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, 1)) + + def up(t, dtype): + return ttnn.from_torch( + t.reshape(sp, tp, rows, -1), + device=mesh_device, + dtype=dtype, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=shard, + ) + + dev_k = ttnn.from_torch( + k[None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + dev_ids = up(ids.to(torch.int32), ttnn.uint32) # the sentinel wraps to -1: its uint32 bits + args = ( + up(q.reshape(seq, -1), ttnn.bfloat16), + up(w, ttnn.bfloat16), + dev_k, + dev_ids, + _pin_table(mesh_device, rows), + ) + out = candidate_scores(*args, start, t_rows) + assert list(out.shape) == [1, 1, rows, nk * 8] + dev = _per_query(out)[0, 0].float() + again = _per_query(candidate_scores(*args, start, t_rows))[0, 0].float() + finite = torch.isfinite(ref) + assert torch.equal(torch.isfinite(dev), finite), (torch.isfinite(dev) != finite).nonzero()[:8].tolist() + diff = (dev[finite] - ref[finite]).abs() + exact = float((diff == 0).float().mean()) + pcc = comp_pcc(ref[finite], dev[finite], 0.0)[1] + print(f"candidate_scores K{nk}: exact {exact:.4f}, max abs diff {float(diff.max()):.3e}, pcc {pcc:.6f}") + # the device's head sum is not an exact fp32 sum (measured: within 2^-7.6 of sum_h |terms| beyond the + # output's + # bf16 rounding); bound it by one bf16 rounding of the output plus 2^-6 of the terms' magnitude + bound = ref[finite].abs() * 2**-7 + magnitude[finite] * 2**-6 + 1e-6 + assert bool((diff <= bound).all()), float((diff / bound).max()) + assert pcc >= 0.9999, pcc + assert torch.equal(again, dev), "candidate_scores is not deterministic" + # the per-row pin page: each row's own position (here: the chunk's, so the same finite pattern), then a + # row + # whose entry is the sentinel sees every candidate row, and rows with earlier positions see fewer + with phase("per-row pins", what="device candidate scores with a per-row pin page"): + newest = positions - start + dev2 = _per_query(candidate_scores(*args[:4], _pin_rows(mesh_device, newest, rows), start, t_rows)) + assert torch.equal(dev2[0, 0].float(), dev), "the per-row page with the chunk's positions differs" + newest = newest.clone() + newest[::5] = SENTINEL + newest[1::5] = (newest[1::5] - 37).clamp(min=0) + ref2, magnitude2 = _candidate_score_ref(q, w, k, ids, torch.where(newest == SENTINEL, 1 << 40, start + newest)) + dev2 = _per_query(candidate_scores(*args[:4], _pin_rows(mesh_device, newest, rows), start, t_rows)) + dev2 = dev2[0, 0].float() + finite2 = torch.isfinite(ref2) + assert torch.equal(torch.isfinite(dev2), finite2), (torch.isfinite(dev2) != finite2).nonzero()[:8].tolist() + assert torch.equal(dev2[finite2 & finite], dev[finite2 & finite]), "visible scores changed with the pin page" + diff2 = (dev2[finite2] - ref2[finite2]).abs() + bound2 = ref2[finite2].abs() * 2**-7 + magnitude2[finite2] * 2**-6 + 1e-6 + assert bool((diff2 <= bound2).all()), float((diff2 / bound2).max()) + # the index-K as persistent block pages (the decode pool's form), the queried slot's block 0 at a page + # offset + with phase("block pages", what="device candidate scores from block pages at a base"): + base_blocks = 3 * 256 # another slot's pages before this one's + pages = torch.cat( + [torch.randn(base_blocks * 8, 128).to(torch.bfloat16), k, torch.zeros(32, 128).to(torch.bfloat16)] + ) + dev_pages = ttnn.from_torch( + pages.reshape(1, 1, -1, 8 * 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + dev3 = _per_query( + candidate_scores(args[0], args[1], dev_pages, args[3], args[4], start, t_rows, block_base=base_blocks) + ) + assert torch.equal(dev3[0, 0].float(), dev), "the block pages at a base differ from the row-major cache" + + +def _range_score_ref(q_u: torch.Tensor, w_u: torch.Tensor, k_u: torch.Tensor, newest: int): + """[W] reference of ``range_scores`` for one user (``_candidate_score_ref``'s arithmetic over every key): + per + head a bf16 q . k, relu, weighted sum over the heads in fp32, one bf16 rounding; -inf past ``newest`` + (every + key for the sentinel). Also returns sum_h |w_h relu(q_h . k)|.""" + s = torch.einsum("hd,td->ht", q_u.float(), k_u.float()).to(torch.bfloat16).float().relu() + terms = w_u.float().unsqueeze(-1) * s + out, magnitude = terms.sum(0).to(torch.bfloat16).float(), terms.abs().sum(0) + visible = torch.arange(k_u.shape[0]) <= (newest if newest != SENTINEL else -1) + return torch.where(visible, out, float("-inf")), magnitude + + +def _unsplit(rows: torch.Tensor, width: int) -> torch.Tensor: + """A split-layout score [R, Wr] (``split_geometry``) -> the one row [width] of global key order.""" + n_rows, row_width = rows.shape + cw = RANGE_CHUNK * 32 + per_row = row_width // cw + # (r, slot) -> chunk slot * R + r; chunk c holds keys [c * cw, (c + 1) * cw) + chunks = rows.view(n_rows, per_row, cw).permute(1, 0, 2).reshape(-1) # chunk-major, then rows, then keys + assert bool((chunks[width:] == float("-inf")).all()), "the slots past the last chunk are not -inf" + return chunks[:width] + + +def _valid_topk(picked: set, scores: torch.Tensor, k: int) -> bool: + """``picked`` is a top-k of ``scores`` [W] up to ties: the right count and no pick below the k-th value.""" + finite = int(torch.isfinite(scores).sum()) + n = min(k, finite) + if len(picked) != n: + return False + if n == 0: + return True + kth = float(scores.topk(n).values[-1]) + return all(float(scores[t]) >= kth for t in picked) + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("split", [1, 32], ids=lambda r: f"rows{r}") +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_range_scores(mesh_device, device_params, split): + """``range_scores`` (the single-query scorer) vs torch on every chip: four users of a + 20,480-row pool at a position that is a multiple of neither 32 nor 8, below INDEX_TOPK, on a tile + boundary, + idle (sentinel) and at the full pool, in the one-row and the 32-row split layouts; the finite pattern + exact, the values within ``candidate_scores``'s bound, + deterministic, bit-equal to ``candidate_scores`` over the same keys from block pages; then the one-row + selection + (``TtV41Indexer.select`` with a one-entry pin page) on the device score is a top-k of it up to ties, the + consumer's inside the published blocks, and the dense path (``indexer_score_dsa`` + mask) agrees on the + selection up to the scorers' rounding (the overlap is printed; at least 90 % is demanded).""" + cfg, rows = C, 32 + cap = 20480 + width = cap + 32 + users = 5 + g = torch.Generator().manual_seed(58) + newest = torch.full((rows,), -1, dtype=torch.int64) # the sentinel's bits + newest[:users] = torch.tensor([19999, 299, 4095, -1, cap - 1]) + with phase("inputs", what="FP4-like query / index-K, head weights"): + q = _fp4_like((rows, 32, 128), g) + w = (torch.randn(rows, 32, generator=g) * 0.05).to(torch.bfloat16) + k = _fp4_like((users * width, 128), g) + k.view(users, width, 128)[:, cap:] = 0 # the pad tile is never written + rep = ttnn.ReplicateTensorToMesh(mesh_device) + dev_q = ttnn.from_torch( + q.reshape(1, 1, rows, -1), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + dev_w = ttnn.from_torch( + w.reshape(1, 1, rows, 32), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + dev_k = ttnn.from_torch( + k.reshape(users, 1, width, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + dev_pin = _replicated(mesh_device, newest.view(1, 1, 1, rows).to(torch.int32), ttnn.uint32) + source = TtV41Indexer( + mesh_device, + cfg, + cfg.CANDIDATE_SOURCE_LAYER, + { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + }, + ) + consumer = TtV41Indexer( + mesh_device, + cfg, + 24, + { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + }, + ) + topk = cfg.INDEX_TOPK + overlaps = [] + for u in range(users): + n_u = int(newest[u]) if newest[u] >= 0 else SENTINEL + k_u = k.view(users, width, 128)[u] + with phase(f"user{u}", what=f"range_scores of user {u} (newest {n_u})"): + out = range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split) + n_rows, _, _, row_width = split_geometry(width, split) + assert n_rows == split, (n_rows, split) + assert list(out.shape) == [1, 1, n_rows, row_width], (tuple(out.shape), n_rows, row_width) + devs = [_unsplit(ttnn.to_torch(d)[0, 0].float(), width) for d in ttnn.get_device_tensors(out)] + dev = devs[0] + for i, other in enumerate(devs[1:], 1): + assert torch.equal(other, dev), f"chip {i} differs from chip 0" + again = _unsplit(_chip0(range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split)).float(), width) + assert torch.equal(again, dev), "range_scores is not deterministic" + ref, magnitude = _range_score_ref(q[u], w[u], k_u, n_u) + finite = torch.isfinite(ref) + assert torch.equal(torch.isfinite(dev), finite), ( + (torch.isfinite(dev) != finite).nonzero()[:8].flatten().tolist() + ) + if n_u == SENTINEL: + print(f"range_scores user {u}: idle, all -inf") + continue + diff = (dev[finite] - ref[finite]).abs() + bound = ref[finite].abs() * 2**-7 + magnitude[finite] * 2**-6 + 1e-6 + pcc = comp_pcc(ref[finite], dev[finite], 0.0)[1] + print( + f"range_scores user {u} (newest {n_u}): exact {float((diff == 0).float().mean()):.4f}, max abs diff {float(diff.max()):.3e}, pcc {pcc:.6f}" + ) + assert bool((diff <= bound).all()), float((diff / bound).max()) + assert pcc >= 0.9999, pcc + # the same arithmetic as candidate_scores over the same keys (every block in order, from block pages) + with phase(f"user{u}-cand", what="candidate_scores over every block"): + nb = width // 8 + pages = ttnn.from_torch( + k_u.reshape(1, 1, nb, 8 * 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + ids = _replicated(mesh_device, torch.arange(nb, dtype=torch.int32).view(1, 1, 1, nb), ttnn.uint32) + q1 = ttnn.from_torch( + q[u].reshape(1, 1, 1, -1), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + w1 = ttnn.from_torch( + w[u].reshape(1, 1, 1, 32), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + pin1 = ttnn.slice(dev_pin, [0, 0, 0, u], [1, 1, 1, u + 1]) + cand = _chip0(candidate_scores(q1, w1, pages, ids, pin1, 0, width))[0].float() + assert torch.equal(cand, dev), f"candidate_scores differs at {(cand != dev).nonzero()[:8].flatten().tolist()}" + # the split selection on the device score (the rows merged), with the user's one-entry pin page + with phase(f"user{u}-select", what="split selection"): + idx, published = source.select_split(out, pin1, width, topk) + picks = _rows(_chip0(idx))[0] + blocks = None if published.ids is None else _rows(_chip0(published.ids))[0] + assert published.ids is not None and list(published.ids.shape) == [1, 1, 1, cfg.CANDIDATE_TOPK_BLOCKS] + cidx = consumer.select_user_in_blocks(dev_q, dev_w, pages, pin1, published.ids, u, width, 0, topk) + cpicks = _rows(_chip0(cidx))[0] + assert _valid_topk(picks, dev, topk), f"user {u}: the source's picks are not a top-{topk}" + mask = v41.select_candidate_blocks( + dev[None].clone(), n_u + 1, cfg.CANDIDATE_TOPK_BLOCKS, cfg.CANDIDATE_BLOCK_SIZE + ) + assert blocks is not None + ref_blocks = _ref_blocks(mask, cfg.CANDIDATE_BLOCK_SIZE)[0] + if blocks != ref_blocks: + # ties among block maxima at the boundary: every device block must reach the reference's k-th + # maximum + bmax = torch.nn.functional.pad(dev, (0, -width % 8), value=float("-inf")).view(-1, 8).amax(-1) + kth = min(float(bmax[b]) for b in ref_blocks if b != n_u // 8) + extra = sorted(blocks - ref_blocks)[:8] + missing = sorted(ref_blocks - blocks)[:8] + detail = f"{len(blocks)} vs {len(ref_blocks)} blocks, kth {kth}, extra {[(b, float(bmax[b])) for b in extra]}, missing {[(b, float(bmax[b])) for b in missing]}" + assert len(blocks) == len(ref_blocks) and all( + float(bmax[b]) >= kth for b in blocks if b != n_u // 8 + ), f"user {u}: blocks {detail}" + assert _valid_topk(cpicks, dev.masked_fill(~mask[0], float("-inf")), topk), f"user {u}: the consumer's picks" + # the dense reference path: indexer_score_dsa on the 32-row tile + the mask + with phase(f"user{u}-dense", what="indexer_score_dsa + mask"): + qt = ttnn.from_torch( + q.permute(1, 0, 2).reshape(1, 32, rows, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + wt = ttnn.from_torch( + w.reshape(1, 1, rows, 32), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + config = ttnn.IndexerScoreProgramConfig(q_chunk_size=32, k_chunk_size=min(256, width), head_group_size=0) + dense = ttnn.experimental.indexer_score_dsa( + qt, dev_k, wt, chunk_start_idx=cap, seq_shard_axes=[], program_config=config, cache_batch_idx=u + ) + dense_u = _chip0(dense)[u].float() + dense_u[n_u + 1 :] = float("-inf") + fin = torch.isfinite(dense_u) & finite + pcc_dense = comp_pcc(dense_u[fin], dev[fin], 0.0)[1] + dense_top = set(dense_u.topk(min(topk, n_u + 1)).indices.tolist()) + overlap = len(picks & dense_top) / max(1, len(picks)) + overlaps.append(overlap) + print(f"range_scores user {u}: vs indexer_score_dsa pcc {pcc_dense:.6f}, top-{topk} overlap {overlap:.4f}") + assert overlap >= 0.9, overlap + # timing of the full-pool user: the launch and the split selection (with the host's dispatch) + with phase("timing", what="range_scores and select_split at the full pool"): + u = users - 1 + pin1 = ttnn.slice(dev_pin, [0, 0, 0, u], [1, 1, 1, u + 1]) + out = range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split) + for what, fn in ( + ("range_scores", lambda: range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split)), + ("select_split (source)", lambda: source.select_split(out, pin1, width, topk)), + ("select_split (plain)", lambda: consumer.select_split(out, pin1, width, topk, CandidateBlocks(None))), + ): + fn() + ttnn.synchronize_device(mesh_device) + t0 = time.perf_counter() + for _ in range(10): + fn() + ttnn.synchronize_device(mesh_device) + print(f"{what} at {cap} keys: {(time.perf_counter() - t0) / 10 * 1e3:.3f} ms a call (with host)") + print(f"range_scores overlaps with the dense path: {overlaps}") + + +@pytest.mark.parametrize("split", [1, 32], ids=lambda r: f"rows{r}") +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_select_split_rows(mesh_device, device_params, split): + """``TtV41Indexer.select_split_rows`` (the live rows' selections merged in one chain) on + four tile rows' split-layout scores stacked along the rows, against ``select_split`` row by row: the top-k + ids + and the candidate source's published blocks bit-equal, for the candidate source and for a plain consumer, + in + the one-row and the 32-row layouts; the rows at distinct positions (one below INDEX_TOPK, one idle). Then + the + per-row chain and the merged one are timed over four rows.""" + cfg, rows = C, 32 + cap = 20480 + width = cap + 32 + users = 4 + g = torch.Generator().manual_seed(135) + newest = torch.full((rows,), -1, dtype=torch.int64) + newest[:users] = torch.tensor([19999, 299, 4095, -1]) + tile_rows = [0, 4, 5, 6] # a verify block's rows under the c1 MoE: the user's own, then the rows past the users' + with phase("inputs", what="FP4-like query / index-K, head weights"): + q = _fp4_like((rows, 32, 128), g) + w = (torch.randn(rows, 32, generator=g) * 0.05).to(torch.bfloat16) + k = _fp4_like((users * width, 128), g) + k.view(users, width, 128)[:, cap:] = 0 + rep = ttnn.ReplicateTensorToMesh(mesh_device) + dev_q = ttnn.from_torch( + q.reshape(1, 1, rows, -1), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + dev_w = ttnn.from_torch( + w.reshape(1, 1, rows, 32), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=rep, + ) + dev_k = ttnn.from_torch( + k.reshape(users, 1, width, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + # the tile's pin page: row t's newest (the verify rows read their user's keys with their own pin + # entry) + page = torch.full((rows,), -1, dtype=torch.int64) + for u, t in enumerate(tile_rows): + page[t] = newest[u] + dev_pin = _replicated(mesh_device, page.view(1, 1, 1, rows).to(torch.int32), ttnn.uint32) + weights = { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + } + source = TtV41Indexer(mesh_device, cfg, cfg.CANDIDATE_SOURCE_LAYER, weights) + consumer = TtV41Indexer(mesh_device, cfg, 24, weights) + topk = cfg.INDEX_TOPK + with phase("scores", what="range_scores of the four rows (user u's keys, tile row t's query and pin)"): + scores = [ + range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split, row=t) for u, t in enumerate(tile_rows) + ] + n_rows = scores[0].shape[2] + assert n_rows == split, (n_rows, split) + stacked = ttnn.concat(scores, dim=2) + idx = _replicated( + mesh_device, + torch.tensor([t for t in tile_rows for _ in range(n_rows)], dtype=torch.int32).view(1, 1, 1, -1), + ttnn.uint32, + ) + wide = ttnn.concat([dev_pin] * (-(-idx.shape[3] // rows)), dim=3) + newest_page = ttnn.gather(wide, -1, idx) + got = [int(v) for v in ttnn.to_torch(ttnn.get_device_tensors(newest_page)[0])[0, 0, 0].tolist()] + want = [int(page[t]) & 0xFFFFFFFF for t in tile_rows for _ in range(n_rows)] + assert got == want, (got[:8], want[:8]) + for name, indexer, cand in (("source", source, None), ("consumer", consumer, CandidateBlocks(None))): + with phase(f"{name}-merged", what="select_split_rows over the stacked rows"): + merged, published = indexer.select_split_rows(stacked, newest_page, users, width, topk, cand) + assert list(merged.shape) == [1, 1, users, topk], tuple(merged.shape) + merged_t = _chip0(merged) + blocks_t = None if published is None or published.ids is None else _chip0(published.ids) + for u, t in enumerate(tile_rows): + with phase(f"{name}-row{u}", what="select_split of the row alone"): + pin_t = ttnn.slice(dev_pin, [0, 0, 0, t], [1, 1, 1, t + 1]) + one, pub = indexer.select_split(scores[u], pin_t, width, topk, cand) + one_t = _chip0(one)[0] + assert torch.equal( + merged_t[u], one_t + ), f"{name}: row {u}'s top-k differs at {(merged_t[u] != one_t).nonzero()[:8].flatten().tolist()}" + if blocks_t is not None: + pub_t = _chip0(pub.ids)[0] + assert torch.equal(blocks_t[u], pub_t), f"{name}: row {u}'s blocks differ" + n_valid = len(_rows(one_t.view(1, -1))[0]) + print( + f"select_split_rows {name} row {u} (newest {int(newest[u])}): {n_valid} picks, bit-equal to the row's own" + ) + with phase("timing", what="the per-row chain against the merged one, four rows (with the host's dispatch)"): + pins = [ttnn.slice(dev_pin, [0, 0, 0, t], [1, 1, 1, t + 1]) for t in tile_rows] + for what, fn in ( + ( + "select_split x 4 (source)", + lambda: [source.select_split(s, p, width, topk) for s, p in zip(scores, pins)], + ), + ("select_split_rows (source)", lambda: source.select_split_rows(stacked, newest_page, users, width, topk)), + ( + "select_split x 4 (plain)", + lambda: [consumer.select_split(s, p, width, topk, CandidateBlocks(None)) for s, p in zip(scores, pins)], + ), + ( + "select_split_rows (plain)", + lambda: consumer.select_split_rows(stacked, newest_page, users, width, topk, CandidateBlocks(None)), + ), + ): + fn() + ttnn.synchronize_device(mesh_device) + t0 = time.perf_counter() + for _ in range(10): + fn() + ttnn.synchronize_device(mesh_device) + print( + f"{what} at {cap} keys, {split} split rows: {(time.perf_counter() - t0) / 10 * 1e3:.3f} ms a call (with host)" + ) + + +_LIVE_POSITIONS = [19999, 299, 4095, -1, 20479, 8191, 12345, 16383, 20000, 511, 17, 9000, 2047, -1, 15000, 6001] + + +def _live_layout(layout: str, users: int = 4) -> list[tuple[int, int]]: + """(tile row, user) of the live rows, as ``TtV41VerifyDecoder.block_rows`` lays them out at K = 3: ``c1`` + one + user's block (its own row, then the rows past the users'), ``c32`` every user's four rows.""" + if layout == "c1": + return [(3, 3), (4, 3), (5, 3), (6, 3)] + return [(u * 4 + j, u) for u in range(users) for j in range(4)] + + +def _blocks_case(mesh_device, layout: str) -> dict: + """The inputs of a consumer's selection over the live rows of ``layout`` on a pool of four users of 20,480 + rows: + FP4-like query and index-K block pages, head weights, the tile's pin page (the rows at distinct positions, + one + below INDEX_TOPK, idle ones) and the blocks published for each live row (the pinned block first, random + blocks, + a short list with a sentinel tail every fourth row, sentinels for an idle row).""" + cfg, rows, cap, users = C, 32, 20480, 4 + width = cap + 32 + block, kb = cfg.CANDIDATE_BLOCK_SIZE, cfg.CANDIDATE_TOPK_BLOCKS + nb = width // block + g = torch.Generator().manual_seed(259) + live_rows = _live_layout(layout, users) + n_live = len(live_rows) + positions = _LIVE_POSITIONS[:n_live] + newest = torch.full((rows,), -1, dtype=torch.int64) # the sentinel's bits + ids = torch.full((n_live, kb), -1, dtype=torch.int64) + for l, (t, _) in enumerate(live_rows): + newest[t] = positions[l] + if positions[l] < 0: + continue # an idle row: the candidate source published sentinels + pinned = positions[l] // block + others = torch.randperm(cap // block, generator=g) + others = others[others != pinned] + valid = kb if l % 4 != 2 else kb // 3 # a short list: the sentinel tail of a row with few reachable blocks + ids[l, 0] = pinned + ids[l, 1:valid] = others[: valid - 1] + q = _fp4_like((rows, 32, 128), g) + w = (torch.randn(rows, 32, generator=g) * 0.05).to(torch.bfloat16) + k = _fp4_like((users * width, 128), g) + k.view(users, width, 128)[:, cap:] = 0 # the pad tile is never written + rep = ttnn.ReplicateTensorToMesh(mesh_device) + row_major = dict(device=mesh_device, dtype=ttnn.bfloat16, layout=ttnn.ROW_MAJOR_LAYOUT, mesh_mapper=rep) + consumer = TtV41Indexer( + mesh_device, + cfg, + 24, + { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + }, + ) + return { + "cap": cap, + "width": width, + "users": users, + "nb": nb, + "live_rows": live_rows, + "positions": positions, + "ids": ids, + "k": k, + "q": ttnn.from_torch(q.reshape(1, 1, rows, -1), **row_major), + "w": ttnn.from_torch(w.reshape(1, 1, rows, 32), **row_major), + "pages": ttnn.from_torch(k.reshape(1, 1, users * nb, block * 128), **row_major), + "pin": _replicated(mesh_device, newest.view(1, 1, 1, rows).to(torch.int32), ttnn.uint32), + "dev_ids": _replicated(mesh_device, ids.view(1, 1, n_live, kb).to(torch.int32), ttnn.uint32), + "consumer": consumer, + } + + +def _blocks_merged(case: dict): + live = [(t, u * case["nb"]) for t, u in case["live_rows"]] + return case["consumer"].select_rows_in_blocks( + case["q"], case["w"], case["pages"], case["pin"], case["dev_ids"], live, case["cap"], C.INDEX_TOPK + ) + + +def _blocks_row(case: dict, l: int): + t, u = case["live_rows"][l] + kb = C.CANDIDATE_TOPK_BLOCKS + pin_t = ttnn.slice(case["pin"], [0, 0, 0, t], [1, 1, 1, t + 1]) + ids_t = ttnn.slice(case["dev_ids"], [0, 0, l, 0], [1, 1, l + 1, kb]) + return case["consumer"].select_user_in_blocks( + case["q"], case["w"], case["pages"], pin_t, ids_t, t, case["cap"], u * case["nb"], C.INDEX_TOPK + ) + + +@pytest.mark.parametrize("layout", ["c1", "c32"]) +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_select_rows_in_blocks(mesh_device, device_params, layout): + """``TtV41Indexer.select_rows_in_blocks`` (the live rows' selections in their candidate blocks in one + chain) against ``select_user_in_blocks`` row by row, bit-equal on every chip: the verify step's two + layouts on a pool of four users of 20,480 rows (``c1``: one user's block of four tile rows, its own and + the rows + past the users'; ``c32``: every user's four rows, 16 live rows), the rows at distinct positions (one below + INDEX_TOPK, one idle with sentinel blocks, one at the pool's end, one with a short block list). Then the + per-row + chain and the merged one are timed.""" + block = C.CANDIDATE_BLOCK_SIZE + with phase("inputs", what="FP4-like query / index-K block pages, head weights, pins, published blocks"): + case = _blocks_case(mesh_device, layout) + live_rows, positions, ids = case["live_rows"], case["positions"], case["ids"] + n_live = len(live_rows) + with phase("merged", what=f"select_rows_in_blocks over {n_live} live rows"): + merged = _blocks_merged(case) + assert list(merged.shape) == [1, 1, n_live, C.INDEX_TOPK], tuple(merged.shape) + chips = [ttnn.to_torch(d)[0, 0] for d in ttnn.get_device_tensors(merged)] + merged_t = chips[0] + for i, other in enumerate(chips[1:], 1): + assert torch.equal(other, merged_t), f"chip {i} differs from chip 0" + again = _chip0(_blocks_merged(case)) + assert torch.equal(again, merged_t), "select_rows_in_blocks is not deterministic" + total = 0 + for l, (t, u) in enumerate(live_rows): + with phase(f"row{l}", what="select_user_in_blocks of the row alone"): + one_t = _chip0(_blocks_row(case, l))[0] + differ = (merged_t[l] != one_t).nonzero()[:8].flatten().tolist() + assert torch.equal(merged_t[l], one_t), f"live row {l} (tile row {t}, user {u}): the top-k differs at {differ}" + picks = _rows(one_t.view(1, -1))[0] + if positions[l] >= 0: + assert picks and max(picks) <= positions[l], (l, len(picks), max(picks, default=None), positions[l]) + allowed = {int(b) for b in ids[l].tolist() if b >= 0} + assert all(p // block in allowed for p in picks), f"live row {l}: a pick outside its published blocks" + total += len(picks) + print( + f"select_rows_in_blocks {layout} live row {l} (tile row {t}, user {u}, newest {positions[l]}): " + f"{len(picks)} picks, bit-equal to the row's own" + ) + assert total > 0 + with phase("timing", what="the per-row chain against the merged one (with the host's dispatch)"): + for what, fn in ( + (f"select_user_in_blocks x {n_live}", lambda: [_blocks_row(case, l) for l in range(n_live)]), + ("select_rows_in_blocks", lambda: _blocks_merged(case)), + ): + fn() + ttnn.synchronize_device(mesh_device) + t0 = time.perf_counter() + for _ in range(10): + fn() + ttnn.synchronize_device(mesh_device) + ms = (time.perf_counter() - t0) / 10 * 1e3 + print(f"{what} ({layout}, {n_live} live rows): {ms:.3f} ms a call (with host)") + + +@pytest.mark.parametrize("layout", ["c1", "c32"]) +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_candidate_scores_skip_sentinel(mesh_device, device_params, layout, monkeypatch): + """``V41_CANDSCORE_SENTINEL=skip`` (no read for a sentinel block id) against the default + (the slot's block 0 read for it): the merged selection and every row's own bit-equal between the two, on + rows + with sentinel tails and idle rows of all sentinels; then a launch over idle rows alone (the padding chips + of a + one-slot step) is timed both ways.""" + from models.demos.deepseek_v3_d_p.tt.v41 import indexer_kernels + + with phase("inputs", what="FP4-like query / index-K block pages, head weights, pins, published blocks"): + case = _blocks_case(mesh_device, layout) + n_live = len(case["live_rows"]) + results = {} + for mode in (False, True): + monkeypatch.setattr(indexer_kernels, "CANDSCORE_SKIP_SENTINEL", mode) + with phase(f"skip{int(mode)}", what="the merged selection and the rows' own"): + merged = [ttnn.to_torch(d)[0, 0] for d in ttnn.get_device_tensors(_blocks_merged(case))] + for i, other in enumerate(merged[1:], 1): + assert torch.equal(other, merged[0]), f"skip {mode}: chip {i} differs from chip 0" + results[mode] = (merged[0], [_chip0(_blocks_row(case, l))[0] for l in range(n_live)]) + assert torch.equal(results[True][0], results[False][0]), "the merged selection differs with the sentinels skipped" + for l in range(n_live): + assert torch.equal(results[True][1][l], results[False][1][l]), f"row {l} differs with the sentinels skipped" + assert torch.equal(results[True][0][l], results[True][1][l]), f"row {l}: merged against its own, skipped" + print(f"candidate_scores {layout}: {n_live} live rows bit-equal with the sentinel reads skipped") + # idle rows alone: every id the sentinel, every pin the sentinel + kb = C.CANDIDATE_TOPK_BLOCKS + idle_ids = _replicated(mesh_device, torch.full((1, 1, n_live, kb), -1, dtype=torch.int32), ttnn.uint32) + idle_pin = _replicated(mesh_device, torch.full((1, 1, 1, 32), -1, dtype=torch.int32), ttnn.uint32) + idle = dict(case, dev_ids=idle_ids, pin=idle_pin) + with phase("timing", what="the merged selection over idle rows, both ways (with the host's dispatch)"): + for mode in (False, True): + monkeypatch.setattr(indexer_kernels, "CANDSCORE_SKIP_SENTINEL", mode) + picks = _chip0(_blocks_merged(idle)) + assert bool((picks.long() == SENTINEL).all()), "an idle row picked a key" + ttnn.synchronize_device(mesh_device) + t0 = time.perf_counter() + for _ in range(20): + _blocks_merged(idle) + ttnn.synchronize_device(mesh_device) + ms = (time.perf_counter() - t0) / 20 * 1e3 + print(f"select_rows_in_blocks over {n_live} idle rows, skip {mode}: {ms:.3f} ms a call (with host)") + + +@pytest.mark.parametrize("split", [1, 32], ids=lambda r: f"rows{r}") +@pytest.mark.parametrize("layout", ["c1", "c32"]) +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_range_scores_rows(mesh_device, device_params, layout, split): + """``range_scores(live=)`` (the live rows' scores in one launch that writes their stacked split layouts) + against one launch a row, concatenated: bit-equal on every chip and deterministic, in the + verify step's two layouts (4 and 16 live rows of a four-user pool of 20,480 rows, rows at distinct + positions, + idle ones) and in the one-row and the 32-row split layouts (with 32 split rows the 128 and 512 units go + round + the grid's cores, several a core). Then both forms are timed.""" + rows, cap, users = 32, 20480, 4 + width = cap + 32 + g = torch.Generator().manual_seed(261) + live_rows = _live_layout(layout, users) + n_live = len(live_rows) + newest = torch.full((rows,), -1, dtype=torch.int64) + for l, (t, _) in enumerate(live_rows): + newest[t] = _LIVE_POSITIONS[l] + with phase("inputs", what="FP4-like query / index-K, head weights, pins"): + q = _fp4_like((rows, 32, 128), g) + w = (torch.randn(rows, 32, generator=g) * 0.05).to(torch.bfloat16) + k = _fp4_like((users * width, 128), g) + k.view(users, width, 128)[:, cap:] = 0 + rep = ttnn.ReplicateTensorToMesh(mesh_device) + row_major = dict(device=mesh_device, dtype=ttnn.bfloat16, layout=ttnn.ROW_MAJOR_LAYOUT, mesh_mapper=rep) + dev_q = ttnn.from_torch(q.reshape(1, 1, rows, -1), **row_major) + dev_w = ttnn.from_torch(w.reshape(1, 1, rows, 32), **row_major) + dev_k = ttnn.from_torch( + k.reshape(users, 1, width, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + dev_pin = _replicated(mesh_device, newest.view(1, 1, 1, rows).to(torch.int32), ttnn.uint32) + live = [(u, t) for t, u in live_rows] + + def stacked_rows(): + return range_scores(dev_q, dev_w, dev_k, dev_pin, 0, width, rows=split, live=live) + + def one_row(l: int): + t, u = live_rows[l] + return range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split, row=t) + + n_rows, _, _, row_width = split_geometry(width, split) + assert n_rows == split, (n_rows, split) + with phase("stacked", what=f"range_scores over {n_live} live rows"): + out = stacked_rows() + assert list(out.shape) == [1, 1, n_live * n_rows, row_width], tuple(out.shape) + chips = [ttnn.to_torch(d)[0, 0].float() for d in ttnn.get_device_tensors(out)] + stacked_t = chips[0] + for i, other in enumerate(chips[1:], 1): + assert torch.equal(other, stacked_t), f"chip {i} differs from chip 0" + assert torch.equal(_chip0(stacked_rows()).float(), stacked_t), "range_scores(live=) is not deterministic" + for l, (t, u) in enumerate(live_rows): + with phase(f"row{l}", what="range_scores of the row alone"): + one_t = _chip0(one_row(l)).float() + mine = stacked_t[l * n_rows : (l + 1) * n_rows] + differ = (mine != one_t).nonzero()[:8].tolist() + assert torch.equal(mine, one_t), f"live row {l} (tile row {t}, user {u}): the scores differ at {differ}" + finite = int(torch.isfinite(one_t).sum()) + want = max(0, _LIVE_POSITIONS[l] + 1) + assert finite == want, (l, finite, want) + print(f"range_scores(live=) {layout} live row {l} (tile row {t}, user {u}): {finite} keys, bit-equal") + with phase("timing", what="a launch a row and the concat against the one launch (with the host's dispatch)"): + for what, fn in ( + (f"range_scores x {n_live} + concat", lambda: ttnn.concat([one_row(l) for l in range(n_live)], dim=2)), + ("range_scores(live=)", stacked_rows), + ): + fn() + ttnn.synchronize_device(mesh_device) + t0 = time.perf_counter() + for _ in range(10): + fn() + ttnn.synchronize_device(mesh_device) + ms = (time.perf_counter() - t0) / 10 * 1e3 + print(f"{what} ({layout}, {n_live} live rows, {split} split rows): {ms:.3f} ms a call (with host)") + + +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_range_scores_rows_cache(mesh_device, device_params): + """``range_scores(live=)`` with 4 live rows, then 16, then 4 again in one process, each against one launch + a + row: with 32 split rows both counts launch on the same cores with the same compile-time arguments, so the + second + call is a program-cache hit, and a hit re-applies each core's runtime arguments only up to the length the + first + call gave it (tt_metal apply_descriptor_runtime_args). The 32-slot round's verify step after its plain + step is + that sequence, and hung the device.""" + rows, cap, users, split = 32, 20480, 4, 32 + width = cap + 32 + g = torch.Generator().manual_seed(266) + newest = torch.full((rows,), -1, dtype=torch.int64) + for l, (t, _) in enumerate(_live_layout("c32", users)): + newest[t] = _LIVE_POSITIONS[l] + q = _fp4_like((rows, 32, 128), g) + w = (torch.randn(rows, 32, generator=g) * 0.05).to(torch.bfloat16) + k = _fp4_like((users * width, 128), g) + k.view(users, width, 128)[:, cap:] = 0 + rep = ttnn.ReplicateTensorToMesh(mesh_device) + row_major = dict(device=mesh_device, dtype=ttnn.bfloat16, layout=ttnn.ROW_MAJOR_LAYOUT, mesh_mapper=rep) + dev_q = ttnn.from_torch(q.reshape(1, 1, rows, -1), **row_major) + dev_w = ttnn.from_torch(w.reshape(1, 1, rows, 32), **row_major) + dev_k = ttnn.from_torch( + k.reshape(users, 1, width, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + dev_pin = _replicated(mesh_device, newest.view(1, 1, 1, rows).to(torch.int32), ttnn.uint32) + n_rows = split_geometry(width, split)[0] + for step, layout in enumerate(["c1", "c32", "c1"]): + live_rows = _live_layout(layout, users) + with phase(f"stacked{step}", what=f"range_scores over the {len(live_rows)} live rows of {layout}"): + out = range_scores(dev_q, dev_w, dev_k, dev_pin, 0, width, rows=split, live=[(u, t) for t, u in live_rows]) + ttnn.synchronize_device(mesh_device) + stacked_t = _chip0(out).float() + for l, (t, u) in enumerate(live_rows): + one_t = _chip0(range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, rows=split, row=t)).float() + mine = stacked_t[l * n_rows : (l + 1) * n_rows] + assert torch.equal(mine, one_t), f"call {step} ({layout}), live row {l}: the scores differ" + print(f"range_scores(live=) call {step}: {layout}, {len(live_rows)} live rows bit-equal") + + +@pytest.mark.timeout(900) +@pytest.mark.parametrize("positions", ["lockstep", "spread"]) +@pytest.mark.parametrize("layout", ["c1", "c32"]) +@pytest.mark.parametrize("mesh_device, device_params", KERNEL_MESHES, indirect=True) +def test_v41_range_scores_rows_wide(mesh_device, device_params, layout, positions): + """``range_scores(live=)`` at the production pool's width (a ratio-4 source of the 1,054,720-row pool: + 263,712 + columns, the default 32 split rows of 33 chunk slots each) against one launch a row: the 32-slot verify + gate + hung the device with it, which the narrow pool of ``test_v41_range_scores_rows`` did + not. ``lockstep``: every user near its 75th compressed row and its four rows one apart (the gate's + 300-token + prompt in every slot: one non-empty split row a live row); ``spread``: ``_LIVE_POSITIONS``. Keys past row + 20,480 are zero (only the visible prefix is read).""" + rows, cap, users, prefix = 32, 1054720 // 4, 4, 20480 + width = cap + 32 + g = torch.Generator().manual_seed(263) + live_rows = _live_layout(layout, users) + n_live = len(live_rows) + newest = torch.full((rows,), -1, dtype=torch.int64) + for l, (t, u) in enumerate(live_rows): + j = sum(1 for _, other in live_rows[:l] if other == u) # the row's place in its user's block + newest[t] = 74 + j if positions == "lockstep" else _LIVE_POSITIONS[l] + with phase("inputs", what="FP4-like query / index-K prefix, head weights, pins"): + q = _fp4_like((rows, 32, 128), g) + w = (torch.randn(rows, 32, generator=g) * 0.05).to(torch.bfloat16) + k = torch.zeros(users, width, 128, dtype=torch.bfloat16) + k[:, :prefix] = _fp4_like((users, prefix, 128), g) + rep = ttnn.ReplicateTensorToMesh(mesh_device) + row_major = dict(device=mesh_device, dtype=ttnn.bfloat16, layout=ttnn.ROW_MAJOR_LAYOUT, mesh_mapper=rep) + dev_q = ttnn.from_torch(q.reshape(1, 1, rows, -1), **row_major) + dev_w = ttnn.from_torch(w.reshape(1, 1, rows, 32), **row_major) + dev_k = ttnn.from_torch( + k.reshape(users, 1, width, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + del k + dev_pin = _replicated(mesh_device, newest.view(1, 1, 1, rows).to(torch.int32), ttnn.uint32) + live = [(u, t) for t, u in live_rows] + n_rows, _, per_row, row_width = split_geometry(width) + assert n_rows == 32 and per_row == 33, (n_rows, per_row) + with phase("stacked", what=f"range_scores over {n_live} live rows at width {width}"): + out = range_scores(dev_q, dev_w, dev_k, dev_pin, 0, width, live=live) + ttnn.synchronize_device(mesh_device) + stacked_t = _chip0(out).float() + assert list(stacked_t.shape) == [n_live * n_rows, row_width], tuple(stacked_t.shape) + for l, (t, u) in enumerate(live_rows): + with phase(f"row{l}", what="range_scores of the row alone"): + one_t = _chip0(range_scores(dev_q, dev_w, dev_k, dev_pin, u, width, row=t)).float() + mine = stacked_t[l * n_rows : (l + 1) * n_rows] + differ = (mine != one_t).nonzero()[:8].tolist() + assert torch.equal(mine, one_t), f"live row {l} (tile row {t}, user {u}): the scores differ at {differ}" + finite = int(torch.isfinite(one_t).sum()) + assert finite == max(0, int(newest[t]) + 1), (l, finite, int(newest[t])) + print(f"range_scores(live=) wide {layout} {positions}: {n_live} live rows bit-equal at width {width}") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_fused_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_fused_v41.py new file mode 100644 index 000000000000..5281dd66f07a --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_fused_v41.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Fused V4.1 mHC collapse / hc_post (tt/v41/mhc.py, bead 8y7.9.5) vs the composite ttnn path and fp64 torch. + +The fused ops run the composite path's SFPU multiply / addcmul sequence in the same order, so they must be +bit-identical to ``tt_mhc._mix`` / ``TtMHCWrap.hc_post``; the fp64 comparison guards against both being wrong +the same way. The block's bf16 sublayer interface (bead 8y7.9.9) is checked against the typecasts it replaces: +the bf16 collapse equals the fp32 collapse typecast to bf16, and hc_post of a bf16 ``h`` equals hc_post of that +``h`` typecast to fp32, bit for bit. ``test_mhc_fused_traced_time`` logs the traced per-call time at the production per-chip shape +(chunk 5120 on 2x4: 2560 tokens, hidden slice 1280) against the DRAM bound as ``V41_MHC_PERF`` JSON lines. +""" + +import json +import os +import time +from types import SimpleNamespace + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tt.mhc.tt_mhc import TtMHCWrap, _cols, _mix, _streams +from models.demos.deepseek_v3_d_p.tt.v41.mhc import fused_collapse, fused_hc_post + +N = 4 +DRAM_GBPS = 512.0 # Blackhole p150 DRAM peak (G2 bound) +REPLAYS = 20 +SHAPES = {"small": (128, 256), "two_rows_odd": (64, 96), "production": (2560, 1280)} # (tokens, hidden/tp) + + +def _inputs(tokens, hidden, seed=0): + gen = torch.Generator().manual_seed(seed) + return dict( + x=torch.randn(1, 1, tokens, N * hidden, generator=gen), + h=torch.randn(1, 1, tokens, hidden, generator=gen), + pre=torch.rand(1, 1, tokens, N, generator=gen), + post=2 * torch.rand(1, 1, tokens, N, generator=gen), + comb=torch.rand(1, 1, tokens, N * N, generator=gen) / N, + ) + + +def _upload(device, t): + return ttnn.from_torch(t, dtype=ttnn.float32, layout=ttnn.TILE_LAYOUT, device=device) + + +def _composite(op, d): + if op == "collapse": + return _mix(_streams(d["x"], N), _cols(d["pre"], N)) + return TtMHCWrap.hc_post(SimpleNamespace(n=N), d["h"], d["x"], d["post"], d["comb"]) + + +def _fused(op, d): + if op == "collapse": + return fused_collapse(d["x"], d["pre"], N) + return fused_hc_post(d["h"], d["x"], d["post"], d["comb"], N) + + +def _reference(op, t): + """fp64 torch: V4.1 ``Block.hc_pre`` / ``hc_post`` on the packed layout.""" + s, hidden = t["x"].shape[-2], t["h"].shape[-1] + x = t["x"].double().reshape(s, N, hidden) + if op == "collapse": + return (t["pre"].double().reshape(s, N, 1) * x).sum(1).reshape(1, 1, s, hidden) + comb = t["comb"].double().reshape(s, N, N) # [i, j]: input stream i -> output stream j + y = t["post"].double().reshape(s, N, 1) * t["h"].double().reshape(s, 1, hidden) + y = y + torch.einsum("sij,sid->sjd", comb, x) + return y.reshape(1, 1, s, N * hidden) + + +@pytest.mark.parametrize("shape", list(SHAPES)) +@pytest.mark.parametrize("op", ["collapse", "hc_post"]) +def test_mhc_fused_matches_composite(device, op, shape): + tokens, hidden = SHAPES[shape] + t = _inputs(tokens, hidden) + d = {k: _upload(device, v) for k, v in t.items()} + t0 = time.perf_counter() + fused = [ttnn.to_torch(_fused(op, d)) for _ in range(2)] + logger.info(f"fused {op} {shape}: 2 calls {time.perf_counter() - t0:.2f}s (incl. compile)") + composite = ttnn.to_torch(_composite(op, d)) + assert torch.equal(fused[0], fused[1]), "fused mHC op is not deterministic" + ref = _reference(op, t) + err = (fused[0].double() - ref).abs().max().item() + mismatch = (fused[0] != composite).sum().item() + logger.info(f"{op} {shape}: max |fused - fp64| {err:.3e}, elements != composite {mismatch}") + assert err < 1e-5 * ref.abs().max().item(), err + assert mismatch == 0, f"{mismatch} elements differ from the composite path" + + +def _traced_us(device, fn): + fn() + ttnn.synchronize_device(device) + tid = ttnn.begin_trace_capture(device, cq_id=0) + out = fn() + ttnn.end_trace_capture(device, tid, cq_id=0) + ttnn.execute_trace(device, tid, cq_id=0, blocking=True) + t0 = time.perf_counter() + for _ in range(REPLAYS): + ttnn.execute_trace(device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(device) + us = (time.perf_counter() - t0) / REPLAYS * 1e6 + ttnn.release_trace(device, tid) + out.deallocate() + return us + + +def _fused_bf16(op, d): + """The block's bf16 sublayer interface: collapse -> bf16, hc_post of a bf16 ``h``.""" + if op == "collapse": + return fused_collapse(d["x"], d["pre"], N, ttnn.bfloat16) + return fused_hc_post(d["h_bf16"], d["x"], d["post"], d["comb"], N) + + +def _typecast_bf16(op, d): + """What the bf16 interface replaces: the fp32 fused op with a typecast on the sublayer side.""" + if op == "collapse": + return ttnn.typecast(fused_collapse(d["x"], d["pre"], N), ttnn.bfloat16) + return fused_hc_post(ttnn.typecast(d["h_bf16"], ttnn.float32), d["x"], d["post"], d["comb"], N) + + +def _with_bf16_h(device, d, t): + d["h_bf16"] = ttnn.from_torch( + t["h"].to(torch.bfloat16), dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device + ) + return d + + +@pytest.mark.parametrize("shape", list(SHAPES)) +@pytest.mark.parametrize("op", ["collapse", "hc_post"]) +def test_mhc_fused_bf16_sublayer_io(device, op, shape): + tokens, hidden = SHAPES[shape] + t = _inputs(tokens, hidden) + d = _with_bf16_h(device, {k: _upload(device, v) for k, v in t.items()}, t) + out = [_fused_bf16(op, d) for _ in range(2)] + assert out[0].dtype == (ttnn.bfloat16 if op == "collapse" else ttnn.float32), out[0].dtype + got = [ttnn.to_torch(o) for o in out] + want = ttnn.to_torch(_typecast_bf16(op, d)) + assert torch.equal(got[0], got[1]), "fused mHC op is not deterministic" + mismatch = (got[0] != want).sum().item() + logger.info(f"{op} {shape} bf16 sublayer io: elements != typecast path {mismatch}") + assert mismatch == 0, f"{mismatch} elements differ from the typecast path" + + +@pytest.mark.parametrize("impl", ["fused", "composite", "fused_bf16", "typecast_bf16"]) +@pytest.mark.parametrize("op", ["collapse", "hc_post"]) +@pytest.mark.parametrize("device_params", [{"trace_region_size": 32 << 20}], indirect=True) +def test_mhc_fused_traced_time(device, op, impl): + tokens, hidden = SHAPES["production"] + t = _inputs(tokens, hidden) + d = _with_bf16_h(device, {k: _upload(device, v) for k, v in t.items()}, t) + impls = {"fused": _fused, "composite": _composite, "fused_bf16": _fused_bf16, "typecast_bf16": _typecast_bf16} + us = _traced_us(device, lambda: impls[impl](op, d)) + stream_bytes = 4 * tokens * N * hidden + sub = 0.5 if impl.endswith("bf16") else 1.0 # the sublayer-side tensor (collapse output / h) in bf16 + # collapse: read the streams, write one stream; hc_post: read the streams + h, write the streams + moved = stream_bytes * (1 + sub / N) if op == "collapse" else stream_bytes * (2 + sub / N) + bound_us = moved / (DRAM_GBPS * 1e3) + record = { + "op": op, + "impl": impl, + "tokens": tokens, + "hidden_per_chip": hidden, + "traced_us": round(us, 1), + "dram_bound_us": round(bound_us, 1), + "dram_util": round(bound_us / us, 3), + } + line = "V41_MHC_PERF " + json.dumps(record) + logger.info(line) + with open(os.environ.get("V41_MHC_PERF_OUT", "generated/v41_mhc_perf.log"), "a") as f: + f.write(line + "\n") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_split_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_split_v41.py new file mode 100644 index 000000000000..e336a474e62e --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_split_v41.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Fused V4.1 mHC split projection (tt/v41/mhc.py ``fused_project``, bead 8y7.9.5) vs fp64 torch and the +composite +``tt_mhc._project`` (matmul + x*x + sum, two TP all-reduces). + +Both paths feed the matmul unit fp32 operands (tf32 multiplies, fp32 accumulate), so the fused op is held to +the +composite path's own error against fp64 rather than to bit identity: the sum of squares moves from an SFPU +square + FPU reduce to the diagonal of the FPU Gram tile x @ x^T. ``test_mhc_split_traced_time`` logs the +traced +per-call split time on the 2x4 mesh at the production per-chip shape (chunk 5120: 2560 tokens, hidden slice +1280) +as ``V41_MHC_PERF`` JSON lines. +""" + +import json +import os +import time +from types import SimpleNamespace + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.mhc.tt_mhc import TtMHCWrap, _compute_kernel_config, _project +from models.demos.deepseek_v3_d_p.tt.tt_ccl import per_axis_topology +from models.demos.deepseek_v3_d_p.tt.v41.mhc import _V41Site, fused_project, fused_rms_project, mhc_config + +N = 4 +MIX = (2 + N) * N +EPS = 1e-6 +DRAM_GBPS = 512.0 # Blackhole p150 DRAM peak (G2 bound) +REPLAYS = 20 +REF = SimpleNamespace(norm_eps=C.RMS_NORM_EPS, hc_mult=N, hc_sinkhorn_iters=C.HC_SINKHORN_ITERS, hc_eps=C.HC_EPS) +SHAPES = {"small": (128, 256), "odd": (64, 96), "production": (2560, 1280)} # (tokens, hidden/tp) + + +def _inputs(tokens, hidden, seed=0): + gen = torch.Generator().manual_seed(seed) + k = N * hidden + return torch.randn(1, 1, tokens, k, generator=gen), torch.randn(MIX, k, generator=gen) * k**-0.5 + + +def _upload(device, t): + return ttnn.from_torch(t, dtype=ttnn.float32, layout=ttnn.TILE_LAYOUT, device=device) + + +def _max_err(a, ref): + return (a.double() - ref).abs().max().item() + + +@pytest.mark.parametrize("shape", list(SHAPES)) +def test_mhc_fused_project(device, shape): + tokens, hidden = SHAPES[shape] + x, fn = _inputs(tokens, hidden) + k = x.shape[-1] + fn_pad = torch.cat([fn, torch.zeros(32 - MIX, k)]) + tx, tw, tw24 = ( + _upload(device, x), + _upload(device, fn_pad.t().reshape(1, 1, k, 32)), + _upload(device, fn.t()[None, None]), + ) + + xd, fd = x.double()[0, 0], fn.double() + ref_mixes = xd @ fd.t() + ref_ms = xd.square().mean(-1, keepdim=True) + EPS + ref_proj = ref_mixes * ref_ms.rsqrt() + + t0 = time.perf_counter() + raw = [ttnn.to_torch(fused_project(tx, tw, MIX, k, EPS))[0, 0] for _ in range(2)] + proj = [ttnn.to_torch(fused_rms_project(tx, tw, MIX, EPS))[0, 0] for _ in range(2)] + logger.info(f"fused project {shape}: 4 calls {time.perf_counter() - t0:.2f}s (incl. compile)") + assert torch.equal(raw[0], raw[1]) and torch.equal(proj[0], proj[1]), "fused mHC projection is not deterministic" + + ckc = _compute_kernel_config() + comp_mixes = ttnn.to_torch(ttnn.matmul(tx, tw24, compute_kernel_config=ckc))[0, 0] + comp_ss = ttnn.to_torch(ttnn.sum(ttnn.multiply(tx, tx), dim=-1, keepdim=True))[0, 0, :, :1] + comp_proj = ttnn.to_torch(_project(tx, tw24, EPS, ckc))[0, 0] + + errs = { + "mixes": (_max_err(raw[0][:, :MIX], ref_mixes), _max_err(comp_mixes, ref_mixes)), + "mean_sq": (_max_err(raw[0][:, MIX : MIX + 1], ref_ms), _max_err(comp_ss / k + EPS, ref_ms)), + "projection": (_max_err(proj[0][:, :MIX], ref_proj), _max_err(comp_proj, ref_proj)), + } + logger.info(f"{shape}: max |err| vs fp64 (fused, composite): {errs}") + assert torch.all(raw[0][:, MIX + 1 :] == 0), "mixes columns past the mean of squares must stay zero" + # a K-split projection (a few tile rows: the decode step) sums per-core partials; its mean of squares is + # off by + # up to two tf32 ulps of one lane's partial on a few rows (the + # unsplit kernel is exact to 2e-7); the projection it scales is unchanged to 1e-6 + rows, kt = tokens // 32, k // 32 + nb = kt // max(b for b in range(1, 9) if kt % b == 0) + splits = 1 if rows >= 110 else max(1, min(nb, 110 // rows)) + lane_ulps = float(ref_ms.max()) / 32 / splits * 2**-9 + for key, (fused_err, comp_err) in errs.items(): + bar = max(2 * comp_err + 1e-6, lane_ulps if key == "mean_sq" and splits > 1 else 0.0) + assert fused_err <= bar, (key, errs, bar) + + +def _traced_us(mesh_device, fn): + fn() + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + out = fn() + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=True) + t0 = time.perf_counter() + for _ in range(REPLAYS): + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + us = (time.perf_counter() - t0) / REPLAYS * 1e6 + ttnn.release_trace(mesh_device, tid) + for t in out: + t.deallocate() + return us + + +MESH = pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(trace_region_size=64 << 20), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) + + +def _site_weights(seed=1): + gen = torch.Generator().manual_seed(seed) + width = N * C.EMB_SIZE + return ( + torch.randn(MIX, width, generator=gen) * width**-0.5, + 0.5 * torch.randn(MIX, generator=gen), + 0.5 * torch.randn(3, generator=gen), + ) + + +def _sites(mesh_device, hc): + cfg, topology = mhc_config(C), per_axis_topology()[1] + composite = TtMHCWrap(mesh_device, cfg, *hc, tp_axis=1, topology=topology) + fused = _V41Site(mesh_device, cfg, *hc, tp_axis=1, topology=topology) + return composite, fused + + +def _mesh_streams(mesh_device, tokens, hidden, seed=0): + """-> (host [1, 1, sp * tokens, tp * N * hidden], device: chip (i, j) holds token block i, hidden slice + j).""" + sp, tp = tuple(mesh_device.shape) + gen = torch.Generator().manual_seed(seed) + x = torch.randn(1, 1, sp * tokens, tp * N * hidden, generator=gen) + return x, ttnn.from_torch( + x, + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, tuple(mesh_device.shape), dims=(2, 3)), + ) + + +@MESH +def test_mhc_split_matches_composite_mesh(mesh_device, device_params): + """Production per-chip shape on the mesh: fused and composite split vs the fp32 CPU reference ``hc_mixes`` + (both round the Sinkhorn op's matmul operands to tf32, so they agree to a few tf32 ulps, not bitwise).""" + tokens, hidden = SHAPES["production"] + sp, tp = tuple(mesh_device.shape) + hc = _site_weights() + x_host, x = _mesh_streams(mesh_device, tokens, hidden) + t0 = time.perf_counter() + streams = x_host[0, 0].reshape(sp * tokens, tp, N, hidden).permute(0, 2, 1, 3).reshape(sp * tokens, N * tp * hidden) + ref = v41.Block.hc_mixes(REF, streams.reshape(1, sp * tokens, N, tp * hidden), hc[0], hc[2], hc[1]) + ref = [r[0].flatten(1) for r in ref] # [S, N], [S, N], [S, N * N] + logger.info(f"CPU reference hc_mixes {time.perf_counter() - t0:.1f}s") + composite, fused = _sites(mesh_device, hc) + # split outputs are replicated across TP: keep chip column 0 + down = lambda t: ttnn.to_torch( + t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, tuple(mesh_device.shape), dims=(2, 3)) + )[0, 0] + a = [down(t) for t in composite.split(x)] + b = [down(t) for t in fused.split(x)] + b2 = [down(t) for t in fused.split(x)] + for name, r, u, v, v2 in zip(("pre", "post", "comb"), ref, a, b, b2): + k = r.shape[-1] + assert torch.equal(v, v2), f"fused split {name} is not deterministic" + comp_err, fused_err = _max_err(u[:, :k], r.double()), _max_err(v[:, :k], r.double()) + logger.info(f"split {name}: max |err| vs CPU fused {fused_err:.3e}, composite {comp_err:.3e}") + assert fused_err <= 2 * comp_err + 1e-5, (name, fused_err, comp_err) + + +@pytest.mark.parametrize("impl", ["fused", "composite"]) +@MESH +def test_mhc_split_traced_time(mesh_device, device_params, impl): + tokens, hidden = SHAPES["production"] + _, x = _mesh_streams(mesh_device, tokens, hidden) + composite, fused = _sites(mesh_device, _site_weights()) + site = fused if impl == "fused" else composite + us = _traced_us(mesh_device, lambda: site.split(x)) + bound_us = 4 * tokens * N * hidden / (DRAM_GBPS * 1e3) # read the streams once + record = { + "op": "split", + "impl": impl, + "tokens": tokens, + "hidden_per_chip": hidden, + "traced_us": round(us, 1), + "dram_bound_us": round(bound_us, 1), + "dram_util": round(bound_us / us, 3), + } + line = "V41_MHC_PERF " + json.dumps(record) + logger.info(line) + with open(os.environ.get("V41_MHC_PERF_OUT", "generated/v41_mhc_perf.log"), "a") as f: + f.write(line + "\n") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_v41.py new file mode 100644 index 000000000000..4bc40da75f6d --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_mhc_v41.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 hyper-connections (bead F8): lagged pre-mix chain and final collapse vs the reference. + +Two chained blocks with cheap elementwise sublayers isolate the mHC arithmetic from attention and MoE. The +reference is the vendored ``Block.hc_mixes / hc_pre / hc_post`` with V4.1's lagged wiring and the model's +final ``hc_pre(h, last ffn_pre)``. Bars (G1): mHC >= 0.999, mHC projection >= 0.998; repeats bit-identical. +""" + +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41.mhc import TtV41HyperConnections, initial_pre_mix +from tests.ttnn.utils_for_testing import comp_pcc + +MHC_PCC = 0.999 +PROJECTION_PCC = 0.998 +SEQ = 256 +SNAPSHOT = Path.home() / ( + ".cache/huggingface/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277" +) +REF = SimpleNamespace( + norm_eps=C.RMS_NORM_EPS, hc_mult=C.HC_MULT, hc_sinkhorn_iters=C.HC_SINKHORN_ITERS, hc_eps=C.HC_EPS +) + + +def _synthetic_hc(gen): + mix, width = (2 + C.HC_MULT) * C.HC_MULT, C.HC_MULT * C.EMB_SIZE + return ( + torch.randn(mix, width, generator=gen) * width**-0.5, + 0.5 * torch.randn(mix, generator=gen), + 0.5 * torch.randn(3, generator=gen), + ) + + +def _real_hc(layer): + from safetensors import safe_open + + index = __import__("json").loads((SNAPSHOT / "model.safetensors.index.json").read_text())["weight_map"] + out = {} + for site in ("attn", "ffn"): + names = [f"layers.{layer}.hc_{site}_{k}" for k in ("fn", "base", "scale")] + with safe_open(SNAPSHOT / index[names[0]], "pt") as f: + out[site] = tuple(f.get_tensor(n).float() for n in names) + return out + + +def _ref_block(x, pre_mix, hc, attention, ffn): + """V4.1 Block.forward's residual wiring with the given sublayers (x [S, n, D] bf16).""" + residual = x + a_pre, a_post, a_comb = v41.Block.hc_mixes(REF, x, hc["attn"][0], hc["attn"][2], hc["attn"][1]) + h = attention(v41.Block.hc_pre(REF, x, pre_mix)) + x = v41.Block.hc_post(REF, h, residual, a_post, a_comb) + residual = x + f_pre, f_post, f_comb = v41.Block.hc_mixes(REF, x, hc["ffn"][0], hc["ffn"][2], hc["ffn"][1]) + h = ffn(v41.Block.hc_pre(REF, x, a_pre)) + x = v41.Block.hc_post(REF, h, residual, f_post, f_comb) + return x, f_pre, (a_pre, a_post, a_comb) + + +def _pack(x, tp): + s, n, d = x.shape + return x.reshape(s, n, tp, d // tp).permute(0, 2, 1, 3).reshape(1, 1, s, n * d) + + +def _unpack(t, n, tp): + s, width = t.shape[-2], t.shape[-1] + return t.reshape(s, tp, n, width // n // tp).permute(0, 2, 1, 3).reshape(s, n, -1) + + +def _pcc(a, b): + return comp_pcc(a.float(), b.float(), 0.0)[1] + + +@pytest.mark.parametrize("weights", ["synthetic", "real_layers_2_3"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) +def test_v41_mhc_chain(mesh_device, device_params, weights): + gen = torch.Generator().manual_seed(7) + if weights == "synthetic": + hcs = [{"attn": _synthetic_hc(gen), "ffn": _synthetic_hc(gen)} for _ in range(2)] + else: + if not (SNAPSHOT / "model.safetensors.index.json").is_file(): + pytest.skip("V4.1 checkpoint shards not downloaded") + hcs = [_real_hc(2), _real_hc(3)] + shape, (sp, tp), n = tuple(mesh_device.shape), tuple(mesh_device.shape), C.HC_MULT + + # Sublayers: elementwise scales (distinct per site), identical on both sides. + scales = [torch.randn(C.EMB_SIZE, generator=gen).to(torch.bfloat16) for _ in range(4)] + x0 = torch.randn(SEQ, n, C.EMB_SIZE, generator=gen).to(torch.bfloat16) + + # reference: two blocks, then the model's final collapse + pre = torch.zeros(SEQ, n) + pre[:, 0] = 1.0 + x, ref_pre, ref_mixes = x0[None], pre[None], [] + ref_streams = [] + for b in range(2): + x, ref_pre, mixes = _ref_block( + x, ref_pre, hcs[b], lambda h, s=scales[2 * b]: h * s, lambda h, s=scales[2 * b + 1]: h * s + ) + ref_mixes.append([m[0] for m in mixes]) + ref_streams.append(x[0]) + ref_final = v41.Block.hc_pre(REF, x, ref_pre)[0] + ref_pre = ref_pre[0] + ref_projection = torch.nn.functional.linear(x0.flatten(1).float(), hcs[0]["attn"][0]) * torch.rsqrt( + x0.flatten(1).float().square().mean(-1, keepdim=True) + C.RMS_NORM_EPS + ) + + # device + def tp_row(v): + return ttnn.from_torch( + v.reshape(1, 1, 1, -1), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(None, 3)), + ) + + tt_scales = [tp_row(s) for s in scales] + blocks = [TtV41HyperConnections(mesh_device, C, hc["attn"], hc["ffn"]) for hc in hcs] + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + + def run(): + x = ttnn.from_torch( + _pack(x0.float(), tp), + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + projection = down(blocks[0].attn_site.project(x))[0, 0, :, : ref_projection.shape[-1]] + split = [down(t)[0, 0] for t in blocks[0].attn_site.split(x)] + pre = initial_pre_mix(mesh_device, C, SEQ) + streams = [] + for b, blk in enumerate(blocks): + x, pre = blk( + x, + pre, + attention=lambda h, s=tt_scales[2 * b]: ttnn.multiply(h, s), + ffn=lambda h, s=tt_scales[2 * b + 1]: ttnn.multiply(h, s), + ) + streams.append(_unpack(down(x)[0, 0], n, tp)) + final = down(blocks[-1].final_collapse(x, pre))[0, 0] + return projection, split, streams, down(pre)[0, 0, :, :n], final + + first, second = run(), run() + for a, b in zip(torch.utils._pytree.tree_leaves(first), torch.utils._pytree.tree_leaves(second)): + assert torch.equal(a, b), "mHC chain is not bit-identical across repeats" + projection, split, streams, pre_out, final = first + + tp_cols = lambda t, k: t[:, :k] # split outputs are replicated across TP; keep one copy + results = { + "projection": _pcc(ref_projection, projection), + "pre": _pcc(ref_mixes[0][0], tp_cols(split[0], n)), + "post": _pcc(ref_mixes[0][1], tp_cols(split[1], n)), + "comb": _pcc(ref_mixes[0][2].flatten(-2), tp_cols(split[2], n * n)), + "block0_streams": _pcc(ref_streams[0], streams[0]), + "block1_streams": _pcc(ref_streams[1], streams[1]), + "next_pre_mix": _pcc(ref_pre, pre_out), + "final_collapse": _pcc(ref_final, final), + } + print(f"mHC {weights}: {results}") + assert results["projection"] >= PROJECTION_PCC, results + for key, value in results.items(): + if key != "projection": + assert value >= MHC_PCC, (key, results) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_moe_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_moe_v41.py new file mode 100644 index 000000000000..83315dfd595b --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_moe_v41.py @@ -0,0 +1,145 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 MoE (bead F1) vs the reference ``MoE`` at V4.1 dims. + +The input is the reference's own MoE input (``ffn_in``, the normed residual) of V4.1 layer 2 for one +5120-token chunk, from the §6 oracle; synthetic weights (seeded) or the checkpoint's layer 2. The +reference's gate logits, routing, routed sum, shared expert and output come from the vendored +``Gate`` / ``Expert`` modules (reference numerics: fp32 gate GEMM, FP8 activation QDQ into FP4 routed and +FP8 shared expert GEMMs). Device: ``TtV41Moe`` with bfp8 or bfp4 routed experts (dev-spec D-B: both +dtypes, same bars). + +G1 bars: gate logits >= 0.997, selected scores >= 0.99, top-6 recall >= 0.95, routed >= 0.96, +final >= 0.982, shared expert >= 0.999; bit-identical repeats. + +D-I: ``TtV41Moe`` emulates the reference's FP8 activation QDQ on the expert inputs (the gate reads the +unquantized input, as in the reference); the test also runs the inner ``TtMoe`` without it and reports +both (only the production path is gated). +""" + +import json +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.moe_reference import LAYER, SEQ, device_weights, reference +from models.demos.deepseek_v3_d_p.tt.moe.init_helpers import get_sp_mesh_composer, get_tp_mesh_composer +from models.demos.deepseek_v3_d_p.tt.v41.moe import TtV41Moe +from tests.ttnn.utils_for_testing import comp_pcc + +BARS = {"gate_logits": 0.997, "scores": 0.99, "recall": 0.95, "routed": 0.96, "final": 0.982, "shared": 0.999} +EXPERT_DTYPES = {"bfp8": ttnn.bfloat8_b, "bfp4": ttnn.bfloat4_b} + +MESHES = [ + pytest.param( + shape, + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=shape, topology=f"mesh-{shape[0]}x{shape[1]}"), + id=f"fabric2d-mesh-{shape[0]}x{shape[1]}", + ) + for shape in ((2, 4), (4, 2)) +] + + +def _run(moe: TtV41Moe, mesh_device, x: torch.Tensor, emulate_actq: bool = True) -> dict: + """One forward. ``emulate_actq=False`` bypasses the wrapper's expert-input FP8 QDQ (D-I comparison only).""" + shape = tuple(mesh_device.shape) + tt_x = ttnn.from_torch( + x[None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + start = time.perf_counter() + if emulate_actq: + out, inter = moe(tt_x, return_intermediates=True) + else: + out, inter = moe.moe(ttnn.squeeze(tt_x, dim=0), return_intermediates=True, actual_isl=None) + out = ttnn.unsqueeze(out, dim=0) + ttnn.synchronize_device(mesh_device) + elapsed = time.perf_counter() - start + sp, tp = get_sp_mesh_composer(mesh_device), get_tp_mesh_composer(mesh_device) + k = C.NUM_EXPERTS_PER_TOKEN + host = { + "logits": ttnn.to_torch(inter.gate_logits, mesh_composer=sp).float().reshape(SEQ, -1), + "weights": ttnn.to_torch(inter.gate_scores, mesh_composer=sp).float().reshape(SEQ, k), + "indices": ttnn.to_torch(inter.gate_indices, mesh_composer=sp, dtype=torch.int32).reshape(SEQ, k), + "shared": ttnn.to_torch(inter.shared_output, mesh_composer=tp).float().reshape(SEQ, -1), + "routed": ttnn.to_torch(inter.routed_output, mesh_composer=tp).float().reshape(SEQ, -1), + "final": ttnn.to_torch(out, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))).reshape( + SEQ, -1 + ), + "seconds": elapsed, + } + return host + + +def _pcc(expected, actual) -> float: + return comp_pcc(expected.float(), actual.float(), 0.0)[1] + + +def _metrics(ref: dict, dev: dict) -> dict: + ref_idx, dev_idx = ref["indices"].long(), dev["indices"].long() + hits = (dev_idx[:, :, None] == ref_idx[:, None, :]).any(-1).sum(-1) + k = ref_idx.shape[-1] + return { + "gate_logits": _pcc(ref["logits"], dev["logits"]), + # selected-weight distributions (slot order near score ties is not observable after the reduce) + "scores": _pcc( + ref["weights"].sort(-1, descending=True).values, dev["weights"].sort(-1, descending=True).values + ), + "recall": hits.float().mean().item() / k, + "top6_set_flip_rate": (hits < k).float().mean().item(), + "routed": _pcc(ref["routed"], dev["routed"]), + "shared": _pcc(ref["shared"], dev["shared"]), + "final": _pcc(ref["final"], dev["final"]), + "ms": round(dev["seconds"] * 1e3, 2), + } + + +@pytest.mark.timeout(5400) +@pytest.mark.parametrize("expert_dtype", list(EXPERT_DTYPES)) +@pytest.mark.parametrize("weights_source", ["synthetic", "real"]) +@pytest.mark.parametrize("mesh_device, device_params", MESHES + galaxy_meshes(), indirect=True) +def test_v41_moe(mesh_device, device_params, weights_source, expert_dtype): + ref, model = reference(weights_source) + weights = device_weights(weights_source, model) + del model + moe = TtV41Moe( + mesh_device, + C, + LAYER, + weights, + SEQ, + routed_expert_weights_dtype=EXPERT_DTYPES[expert_dtype], + ) + del weights + + first = _run(moe, mesh_device, ref["x"]) + repeat = _run(moe, mesh_device, ref["x"]) + for name in ("final", "routed", "shared", "logits", "weights", "indices"): + assert torch.isfinite(first[name].float()).all(), f"non-finite {name}" + assert torch.equal(first[name], repeat[name]), f"{name} differs between repeated runs" + first["seconds"] = repeat["seconds"] # warm (program cache) timing + unquantized = _run(moe, mesh_device, ref["x"], emulate_actq=False) + unquantized = _run(moe, mesh_device, ref["x"], emulate_actq=False) # warm timing + + report = { + "mesh": list(mesh_device.shape), + "weights": weights_source, + "expert_dtype": expert_dtype, + "device": _metrics(ref, first), + "device_without_actq_qdq": _metrics(ref, unquantized), + } + logger.info(f"V41_MOE_RESULT {json.dumps(report)}") + failed = {k: v for k, v in report["device"].items() if k in BARS and v < BARS[k]} + assert not failed, f"below G1 bars {BARS}: {failed}; {report}" diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_qdq.py b/models/demos/deepseek_v3_d_p/tests/v41/test_qdq.py new file mode 100644 index 000000000000..d5f482b7c549 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_qdq.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 device quantize-dequantize ops (tt/v41/qdq.py) vs the reference kernels, bit for bit. + +Contract: given the same bf16 input, each device QDQ returns exactly the bf16 values of the reference +``kernel_cpu`` QDQ (``torch.equal``), row-locally on every device of the mesh, deterministically. + +Inputs are built to hit every rounding decision: random groups over ~40 binary decades, all-zero and +tiny groups (amax floors), group amax exactly on / one bf16 ulp around power-of-two scale boundaries, +values exactly on / one ulp around every e4m3 or E2M1 half-way point, e4m3-scale ties and scale +saturation (group-16 FP4), and every finite non-subnormal bf16 value (ordered and shuffled into groups). A small hand-derived golden checks the formulas independently of the +reference. bf16 subnormal inputs are excluded (documented limitation, see qdq.py). +""" + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41.kernel_cpu import act_quant, fp4_act_quant +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41 import qdq + +SEQ = 2048 + +MESH_2X4 = pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", +) + +E4M3_VALUES = torch.arange(256, dtype=torch.uint8).view(torch.float8_e4m3fn).float() +E4M3_VALUES = E4M3_VALUES[torch.isfinite(E4M3_VALUES) & (E4M3_VALUES >= 0)].unique() # 0, 2^-9 ... 448 +E2M1_VALUES = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) + + +def _reference_fp8(x: torch.Tensor) -> torch.Tensor: + return act_quant(x.clone(), qdq.FP8_GROUP, "ue8m0", inplace=True) + + +def _reference_fp4_ue8m0(x: torch.Tensor) -> torch.Tensor: + return fp4_act_quant(x.clone(), qdq.FP4_UE8M0_GROUP, inplace=True) + + +def _reference_fp4_e4m3(x: torch.Tensor) -> torch.Tensor: + return fp4_act_quant(x.clone(), qdq.FP4_E4M3_GROUP, inplace=True, scale_dtype=torch.float8_e4m3fn) + + +# op id -> (device op, reference, group size, format grid of quantized magnitudes, format max) +OPS = { + "fp8_e4m3_g32_ue8m0": (qdq.fp8_qdq, _reference_fp8, qdq.FP8_GROUP, E4M3_VALUES, 448.0), + "fp4_e2m1_g32_ue8m0": (qdq.fp4_ue8m0_qdq, _reference_fp4_ue8m0, qdq.FP4_UE8M0_GROUP, E2M1_VALUES, 6.0), + "fp4_e2m1_g16_e4m3": (qdq.fp4_e4m3_qdq, _reference_fp4_e4m3, qdq.FP4_E4M3_GROUP, E2M1_VALUES, 6.0), +} + + +def _bf16_neighbors(v: torch.Tensor) -> torch.Tensor: + """Positive bf16 values v -> [v, next below, next above] (bf16 bit pattern +-1).""" + bits = v.to(torch.bfloat16).view(torch.int16) + return torch.cat([bits, bits - 1, bits + 1]).view(torch.bfloat16).float() + + +def _tie_pool(grid: torch.Tensor) -> torch.Tensor: + """Grid values, every half-way point between neighbours and their bf16 neighbours (unit scale).""" + midpoints = (grid[1:] + grid[:-1]) / 2 + return _bf16_neighbors(torch.cat([grid[1:], midpoints])).unique() + + +def _finish(gen: torch.Generator, values: torch.Tensor, amax: torch.Tensor) -> torch.Tensor: + """Place amax in a random column, clip other magnitudes to amax, apply random signs.""" + n, group = values.shape + values = torch.minimum(values, amax[:, None]) + values[torch.arange(n), torch.randint(group, (n,), generator=gen)] = amax + signs = torch.where(torch.rand(n, group, generator=gen) < 0.5, -1.0, 1.0) + return values * signs + + +def _random_groups(gen: torch.Generator, n: int, group: int) -> torch.Tensor: + """Normal values with per-group magnitude 2^U(-30, 12); 1/16 all-zero, 1/16 below the amax floors.""" + scale = torch.exp2(torch.empty(n, 1).uniform_(-30, 12, generator=gen)) + values = torch.randn(n, group, generator=gen) * scale + kind = torch.randint(16, (n,), generator=gen) + values[kind == 0] = 0.0 + values[kind == 1] *= 2.0**-100 / scale[kind == 1] # ~2^-100: below every floor, still normal bf16 + return values + + +def _tie_groups(gen: torch.Generator, op: str, n: int) -> torch.Tensor: + """Groups whose scale is exact and known, filled with half-way points of the element format.""" + _, _, group, grid, fmt_max = OPS[op] + if op == "fp4_e2m1_g16_e4m3": + scales = E4M3_VALUES[1:][torch.randint(len(E4M3_VALUES) - 1, (n,), generator=gen)] + else: + # power-of-two scales; the fp8 exponents stay above the 2^-22 floor, the fp4 ones far from subnormals + low, high = (-20, 12) if op == "fp8_e4m3_g32_ue8m0" else (-100, 12) + scales = torch.exp2(torch.randint(low, high, (n,), generator=gen).float()) + # amax = fmt_max * scale sits exactly on the scale boundary (fast_round_scale does not bump; the + # e4m3 scale amax / 6 is exact). + amax = fmt_max * scales + pool = _tie_pool(grid) + picks = pool[torch.randint(len(pool), (n, group), generator=gen)] * scales[:, None] + return _finish(gen, picks, amax) + + +def _scale_boundary_groups(gen: torch.Generator, op: str, n: int) -> torch.Tensor: + """Groups whose amax is on, or one bf16 ulp around, a scale rounding boundary.""" + _, _, group, _, fmt_max = OPS[op] + if op == "fp4_e2m1_g16_e4m3": + # amax / 6 at every e4m3 value and half-way point, including 464 (saturates to 448) and beyond + points = torch.cat([E4M3_VALUES[1:], (E4M3_VALUES[1:] + E4M3_VALUES[:-1]) / 2, torch.tensor([464.0, 1e4])]) + boundaries = points * 6.0 + else: + exponents = torch.arange(-20 if op == "fp8_e4m3_g32_ue8m0" else -110, 14).float() + boundaries = fmt_max * torch.exp2(exponents) + candidates = _bf16_neighbors(boundaries) + amax = candidates[torch.randint(len(candidates), (n,), generator=gen)] + values = torch.rand(n, group, generator=gen) * amax[:, None] + return _finish(gen, values, amax) + + +def _exhaustive_groups(gen: torch.Generator, op: str, n: int, shuffle: bool) -> torch.Tensor: + """Every finite bf16 of both signs above the op's magnitude floor (and the zeros), repeated to fill n groups: in + bit-pattern order (neighbours share a group, so every element is rounded against a comparable amax over the whole + exponent range, bf16 max included; each repeat is rotated to change the group alignment) or shuffled (large + amax). The floor is the smallest normal bf16, or qdq.FP4_UE8M0_MIN_AMAX for the FP4 power-of-two path (see + qdq.py: not reproduced below it).""" + group = OPS[op][2] + floor = qdq.FP4_UE8M0_MIN_AMAX if op == "fp4_e2m1_g32_ue8m0" else 2.0**-126 + values = torch.arange(-(2**15), 2**15, dtype=torch.int32).to(torch.int16).view(torch.bfloat16).float() + values = values[torch.isfinite(values) & ((values == 0) | (values.abs() >= floor))] + parts, filled, rep = [], 0, 0 + while filled < n * group: + part = values[torch.randperm(len(values), generator=gen)] if shuffle else values.roll(7 * rep) + parts.append(part) + filled += len(part) + rep += 1 + return torch.cat(parts)[: n * group].reshape(n, group) + + +def _input(op: str, kind: str, width: int, seed: int = 0) -> torch.Tensor: + group = OPS[op][2] + n = SEQ * width // group + gen = torch.Generator().manual_seed(seed) + if kind == "random": + groups = _random_groups(gen, n, group) + elif kind == "ties": + groups = _tie_groups(gen, op, n) + elif kind in ("exhaustive_ordered", "exhaustive_shuffled"): + groups = _exhaustive_groups(gen, op, n, shuffle=kind == "exhaustive_shuffled") + else: + groups = _scale_boundary_groups(gen, op, n) + x = groups.reshape(1, 1, SEQ, width).to(torch.bfloat16) + assert torch.equal(x.float().reshape(n, group), groups.to(torch.bfloat16).float()) + return x + + +def _to_mesh(x: torch.Tensor, mesh_device, shard: bool) -> ttnn.Tensor: + mapper = ttnn.ShardTensorToMesh(mesh_device, dim=2) if shard else ttnn.ReplicateTensorToMesh(mesh_device) + return ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=mesh_device, mesh_mapper=mapper) + + +def _from_mesh_sharded(t: ttnn.Tensor, mesh_device) -> torch.Tensor: + return ttnn.to_torch(t, mesh_composer=ttnn.ConcatMeshToTensor(mesh_device, dim=2)) + + +def _assert_exact(actual: torch.Tensor, expected: torch.Tensor, x: torch.Tensor) -> None: + assert actual.dtype == torch.bfloat16 and actual.shape == expected.shape + mismatch = actual != expected + if mismatch.any(): + idx = mismatch.nonzero()[:8].tolist() + examples = [(i, x[tuple(i)].item(), actual[tuple(i)].item(), expected[tuple(i)].item()) for i in idx] + raise AssertionError(f"{int(mismatch.sum())} mismatches; (index, input, device, reference): {examples}") + + +@pytest.mark.parametrize("kind", ["random", "ties", "scale_boundaries", "exhaustive_ordered", "exhaustive_shuffled"]) +@pytest.mark.parametrize("width", [512, 128]) +@pytest.mark.parametrize("op", list(OPS)) +@pytest.mark.parametrize("mesh_device, device_params", [MESH_2X4], indirect=True) +def test_qdq_matches_reference(mesh_device, device_params, op, width, kind): + """Sequence-sharded over the mesh (each device quantizes its own rows); exact and deterministic.""" + device_op, reference, *_ = OPS[op] + x = _input(op, kind, width) + expected = reference(x) + tt_x = _to_mesh(x, mesh_device, shard=True) + first = _from_mesh_sharded(device_op(tt_x), mesh_device) + _assert_exact(first, expected, x) + second = _from_mesh_sharded(device_op(tt_x), mesh_device) + assert torch.equal(first, second), "device QDQ is not deterministic across repeated runs" + + +@pytest.mark.parametrize("op", list(OPS)) +@pytest.mark.parametrize("mesh_device, device_params", [MESH_2X4], indirect=True) +def test_qdq_replicated(mesh_device, device_params, op): + """A replicated input yields the reference values on every device.""" + device_op, reference, *_ = OPS[op] + x = _input(op, "random", 512, seed=1) + expected = reference(x) + out = device_op(_to_mesh(x, mesh_device, shard=False)) + for device_out in ttnn.get_device_tensors(out): + _assert_exact(ttnn.to_torch(device_out), expected, x) + + +# Hand-derived goldens (one group per row, rest zero). Each entry: input group -> expected output group. +GOLDEN = { + # amax 448 -> scale 2^ceil(log2(1)) = 1. e4m3 near 17: grid 16, 18 (quantum 2) -> tie 17 -> 16 (even + # mantissa); 19 -> 20. 2^-10 is half of the smallest subnormal 2^-9 -> 0; 3*2^-10 -> 2^-8 (even code 2). + # 1.0625 -> 1 (quantum 1/8 -> tie -> even). + "fp8_e4m3_g32_ue8m0": [ + ([448.0, 17.0, 19.0, 2**-10, 3 * 2**-10, 1.0625, -17.0], [448.0, 16.0, 20.0, 0.0, 2**-8, 1.0, -16.0]), + # amax 450 -> scale 2 (bump); 450/2 = 225 -> 224 (quantum 16 in [128, 256)) -> 448; 3/2 = 1.5 exact + ([450.0, 3.0], [448.0, 3.0]), + # all-zero group -> scale floor 2^-22, output 0 + ([0.0], [0.0]), + ], + # amax 6 -> scale 1; E2M1 ties 0.25 -> 0, 0.75 -> 1, 1.25 -> 1, 1.75 -> 2, 2.5 -> 2, 3.5 -> 4, 5 -> 4. + # amax 6.0625 -> scale 2: 6.0625/2 = 3.03 -> 3 -> 6; 1.4/2 = 0.7 -> 0.5 -> 1. + "fp4_e2m1_g32_ue8m0": [ + ([6.0, 0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, -0.75], [6.0, 0.0, 1.0, 1.0, 2.0, 2.0, 4.0, 4.0, -1.0]), + ([6.0625, 1.40625], [6.0, 1.0]), + ], + # amax 7.5 -> scale e4m3(1.25) = 1.25; 0.9375 = 0.75 s -> 1 s = 1.25; 2.1875 = 1.75 s -> 2 s = 2.5. + # amax 6.375 -> 6.375/6 = 1.0625, e4m3 tie between 1 and 1.125 -> 1 (even); 6.375/1 clamps to 6. + # amax 3008 -> 501.3 saturates to scale 448; 3008/448 = 6.7 -> 6 -> 2688; 900/448 = 2.009 -> 2 -> 896. + # amax bf16(0.01) -> floor 6*2^-9 -> scale 2^-9; 0.01/2^-9 = 5.125 -> 6; 2^-9 = 1 s -> 1 s. + "fp4_e2m1_g16_e4m3": [ + ([7.5, 0.9375, 2.1875], [7.5, 1.25, 2.5]), + ([6.375, 1.5], [6.0, 1.5]), + ([3008.0, 900.0], [2688.0, 896.0]), + ([0.01, 2**-9], [0.01171875, 2**-9]), + ], +} + + +@pytest.mark.parametrize("op", list(OPS)) +@pytest.mark.parametrize("mesh_device, device_params", [MESH_2X4], indirect=True) +def test_qdq_hand_golden(mesh_device, device_params, op): + device_op, reference, group, *_ = OPS[op] + width, rows = 128, 32 * mesh_device.get_num_devices() + x = torch.zeros(rows, width) + expected = torch.zeros(rows, width) + for row, (inputs, outputs) in enumerate(GOLDEN[op]): + x[row, : len(inputs)] = torch.tensor(inputs) + expected[row, : len(outputs)] = torch.tensor(outputs) + x = x.reshape(1, 1, rows, width).to(torch.bfloat16) + expected = expected.reshape(1, 1, rows, width).to(torch.bfloat16) + _assert_exact(reference(x), expected, x) # the golden agrees with the reference + _assert_exact(_from_mesh_sharded(device_op(_to_mesh(x, mesh_device, shard=True)), mesh_device), expected, x) + + +@pytest.mark.parametrize("mesh_device, device_params", [MESH_2X4], indirect=True) +def test_qdq_rejects_unsupported_input(mesh_device, device_params, expect_error): + x = torch.zeros(1, 1, 32, 48, dtype=torch.bfloat16) + with expect_error(AssertionError, "not a multiple of the group size"): + qdq.fp8_qdq(ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=mesh_device)) + with expect_error(AssertionError, "must be bf16"): + qdq.fp8_qdq( + ttnn.from_torch(x[..., :32].float(), dtype=ttnn.float32, layout=ttnn.TILE_LAYOUT, device=mesh_device) + ) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_qdq_perf.py b/models/demos/deepseek_v3_d_p/tests/v41/test_qdq_perf.py new file mode 100644 index 000000000000..be09aa84b1b3 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_qdq_perf.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Traced per-call time of the V4.1 QDQ ops (tt/v41/qdq.py) at production per-chip shapes (bead 8y7.9.4). + +One chip, chunk 5120 on the 4x2 mesh (sp = 4, tp = 2): 1280 rows per chip. Each case captures one call in a +trace and replays it; the reported time is the replay wall time per call (device synchronized), which is what +the traced model pays. The G2 bound is the DRAM roofline of one bf16 read + one bf16 write of the tensor. + +Logged as ``V41_QDQ_PERF`` JSON lines, also appended to ``$V41_QDQ_PERF_OUT`` (default +``generated/v41_qdq_perf.log``); no pass/fail bar (the bead compares before / after against G2). +""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tt.v41 import qdq + +DRAM_GBPS = 512.0 # Blackhole p150 DRAM peak (G2 bound) +REPLAYS = 20 + +# case id -> (qdq function name, per-chip shape) +CASES = { + "fp8_hidden": ("fp8_qdq", (1, 1, 1280, 5120)), + "fp8_q_lora": ("fp8_qdq", (1, 1, 1280, 1280)), + "fp8_window_kv": ("fp8_qdq", (1, 1, 1280, 512)), + "fp4_ue8m0_index_q": ("fp4_ue8m0_qdq", (1, 32, 640, 128)), + "fp4_ue8m0_index_k": ("fp4_ue8m0_qdq", (1, 1, 1280, 128)), + "fp4_e4m3_comp_kv": ("fp4_e4m3_qdq", (1, 1, 640, 512)), +} + + +@pytest.mark.parametrize("case", list(CASES)) +@pytest.mark.parametrize("device_params", [{"trace_region_size": 8 << 20}], indirect=True) +def test_qdq_traced_time(device, case): + name, shape = CASES[case] + fn = getattr(qdq, name) + x = ttnn.from_torch( + torch.randn(shape).to(torch.bfloat16), dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=device + ) + t0 = time.perf_counter() + fn(x) # compile + program cache + ttnn.synchronize_device(device) + compile_s = time.perf_counter() - t0 + + tid = ttnn.begin_trace_capture(device, cq_id=0) + out = fn(x) + ttnn.end_trace_capture(device, tid, cq_id=0) + ttnn.execute_trace(device, tid, cq_id=0, blocking=True) # warm replay + t0 = time.perf_counter() + for _ in range(REPLAYS): + ttnn.execute_trace(device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(device) + per_call_us = (time.perf_counter() - t0) / REPLAYS * 1e6 + ttnn.release_trace(device, tid) + out.deallocate() + + numel = 1 + for d in shape: + numel *= d + bound_us = 4 * numel / (DRAM_GBPS * 1e3) + record = { + "case": case, + "fn": name, + "shape": list(shape), + "traced_us": round(per_call_us, 1), + "dram_bound_us": round(bound_us, 1), + "g2_target_us": round(2 * bound_us, 1), + "dram_util": round(bound_us / per_call_us, 3), + "first_call_s": round(compile_s, 2), + } + line = "V41_QDQ_PERF " + json.dumps(record) + logger.info(line) + with open(os.environ.get("V41_QDQ_PERF_OUT", "generated/v41_qdq_perf.log"), "a") as f: + f.write(line + "\n") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_rms_norm_eps.py b/models/demos/deepseek_v3_d_p/tests/v41/test_rms_norm_eps.py new file mode 100644 index 000000000000..9db46eb70e88 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_rms_norm_eps.py @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 RMSNorm epsilon contract. + +V4.1 sets ``rms_norm_eps=1e-20`` for every RMSNorm (attn/ffn/q/kv/compressor/indexer). +The reference (HF ``inference/model.py`` ``RMSNorm``) computes ``x * rsqrt(mean(x^2) + eps)`` in +fp32. These tests check that the device norms used by deepseek_v3_d_p honor that epsilon: +output matches the fp32 reference, stays finite (including all-zero rows) and repeats +bit-identically. The tiny-magnitude input has mean(x^2) ~ 1e-12, where the V4 default eps +(1e-6) shrinks the output ~1000x. PCC is scale-invariant, so the output magnitude is checked +separately, and the local test runs eps=1e-6 as a control to prove that check is sensitive. +""" + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.tt_ccl import per_axis_topology +from models.demos.deepseek_v3_d_p.tt.tt_distributed_rms_norm import TtDistributedRmsNorm +from tests.ttnn.utils_for_testing import comp_pcc + +V41_RMS_NORM_EPS = 1e-20 +PCC = 0.999 +NORM_RATIO_TOL = 0.02 +SEQ = 256 + +MESH_2X4 = pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", +) + + +def _reference_rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: + x = x.float() + return weight.float() * (x * torch.rsqrt(x.square().mean(-1, keepdim=True) + eps)) + + +def _input(kind: str, dim: int) -> torch.Tensor: + torch.manual_seed(1234) + x = torch.randn(1, 1, SEQ, dim) + if kind == "tiny": + x = x * 1e-6 + elif kind == "zero_rows": + x[..., ::7, :] = 0.0 + return x.to(torch.bfloat16) + + +def _norm_ratio(reference: torch.Tensor, actual: torch.Tensor) -> float: + return (actual.float().norm() / reference.norm()).item() + + +def _check(reference: torch.Tensor, first: torch.Tensor, second: torch.Tensor, kind: str) -> None: + assert torch.isfinite(first).all(), "device RMSNorm produced non-finite values" + assert torch.equal(first, second), "device RMSNorm is not bit-identical across repeated runs" + if kind == "zero_rows": + assert torch.all(first[..., ::7, :] == 0), "all-zero rows must normalize to zero" + passed, pcc = comp_pcc(reference, first.float(), PCC) + assert passed, f"PCC {pcc} < {PCC}" + ratio = _norm_ratio(reference, first) + assert abs(ratio - 1) <= NORM_RATIO_TOL, f"output magnitude ratio {ratio} vs reference" + + +@pytest.mark.parametrize("kind", ["randn", "tiny", "zero_rows"]) +@pytest.mark.parametrize("dim", [128, 512, 1280], ids=["index_k_norm", "kv_norm", "q_norm"]) +@pytest.mark.parametrize("mesh_device, device_params", [MESH_2X4], indirect=True) +def test_local_rms_norm_eps(mesh_device, device_params, dim, kind): + x = _input(kind, dim) + weight = torch.empty(dim).uniform_(0.5, 1.5) + reference = _reference_rms_norm(x, weight, V41_RMS_NORM_EPS) + + replicate = ttnn.ReplicateTensorToMesh(mesh_device) + tt_x = ttnn.from_torch(x, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, device=mesh_device, mesh_mapper=replicate) + tt_w = ttnn.from_torch( + weight.reshape(1, 1, dim // 32, 32).to(torch.bfloat16), + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + device=mesh_device, + mesh_mapper=replicate, + ) + + def run(eps: float = V41_RMS_NORM_EPS) -> torch.Tensor: + out = ttnn.rms_norm(tt_x, weight=tt_w, epsilon=eps) + return ttnn.to_torch(ttnn.get_device_tensors(out)[0]) + + _check(reference, run(), run(), kind) + if kind == "tiny": + control = _norm_ratio(reference, run(eps=1e-6)) + assert abs(control - 1) > NORM_RATIO_TOL, f"eps=1e-6 control ratio {control}: tiny case is insensitive" + + +@pytest.mark.parametrize("kind", ["randn", "tiny", "zero_rows"]) +@pytest.mark.parametrize("mesh_device, device_params", [MESH_2X4], indirect=True) +def test_distributed_rms_norm_eps(mesh_device, device_params, kind): + emb_dim = 5120 + x = _input(kind, emb_dim) + weight = torch.empty(emb_dim).uniform_(0.5, 1.5) + reference = _reference_rms_norm(x, weight, V41_RMS_NORM_EPS) + + mesh_shape = tuple(mesh_device.shape) + norm = TtDistributedRmsNorm( + mesh_device=mesh_device, + emb_dim=emb_dim, + epsilon=V41_RMS_NORM_EPS, + torch_weight=weight, + cluster_axis=1, + num_links=1, + topology=per_axis_topology(device_params["fabric_config"])[1], + ) + tt_x = ttnn.from_torch( + x, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + device=mesh_device, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, mesh_shape=mesh_shape, dims=(None, 3)), + ) + + def run() -> torch.Tensor: + out = norm(tt_x) + full = ttnn.to_torch( + out, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, mesh_shape=mesh_shape, dims=(0, 3)) + ) + return full[:1] + + _check(reference, run(), run(), kind) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_transformer_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_transformer_v41.py new file mode 100644 index 000000000000..8f184e58d2ed --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_transformer_v41.py @@ -0,0 +1,488 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 prefill transformer (bead F9): embed -> layers -> final collapse -> norm -> head, chunked. + +The prompt is real text, so the logits of its last ``SCORED`` positions are teacher-forced next-token +predictions. Free-running V4.1 stacks are chaotic (top-k selection and MoE routing flips: real layer 20's MoE +keeps the same experts on 77% of rows at input PCC 0.9987), so absolute bars on one token's logits measure +flips. + +Gate (user decisions 2026-09-29; playbook ~/knowledge/wiki/playbooks/Acceptance bars.md): each layer's +free-running streams (all rows and the last SCORED rows) vs the reference's single-shot prefill must be no +further than the reference itself drifts when every attention and MoE output carries the error its component +bar +allows and every attention and MoE input the error that flips selection and routing like the device's +(``NOISE``, ``NOISE_SEEDS`` seeds, worst seed), minus ``DRIFT_MARGIN``: the stack composes its components' +errors +no worse than the components predict (a state or composition bug fails it). Top-1 agreement, top-5 recall and +logits PCC over the scored positions are reported against the same floor; repeats are bit-identical. +Cases: small dims (one chunk, two chunks, a padded last chunk), then production shape (S=2048) with synthetic +and real weights, then long context on real weights (bead 8y7.19.1): N real-text chunks of 5120 tokens, the +last +one (e.g. at start 51200 for 11 chunks) attending over the caches the preceding chunks filled on device, with +the +released candidate count (2048 blocks). There the gate also covers the last chunk's rows (``chunk``), and the +block acceptance of test_block_v41 runs at the last chunk: every chunk teacher-forced through every block on a +fresh state (oracle inputs), the last chunk's block outputs >= the real block bar, the KV sources' +compressed-KV and +index-K rows and every window carry >= the cache bar. The single-shot oracle scores the indexer in query +blocks +(``oracle.INDEXER_QUERY_BLOCK``, bit-identical) so that it fits in host memory at S=56320. +""" + +import hashlib +import os +import time +from contextlib import contextmanager + +import pytest +import torch +from loguru import logger + +import ttnn +from models.common import timing_events +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.reference_weights import MOE_KEYS, device_weights +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config, small_spec +from models.demos.deepseek_v3_d_p.tests.v41.test_block_v41 import ( + BLOCK_PCC, + CACHE_PCC, + KV_FORMATS, + _pack, + _pcc, + _unpack, + unrounded_kv, +) +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import ( # noqa: F401 (WEIGHT_CACHE re-export) + WEIGHT_CACHE, + host_weights, + weight_cache_dir, +) +from models.demos.deepseek_v3_d_p.tt.v41.cache import WINDOW_SLOT, V41PrefillState +from models.demos.deepseek_v3_d_p.tt.v41.engram import TtV41Engram, V41EngramHash, V41EngramTable +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer +from models.demos.deepseek_v3_d_p.tt.v41.weights import ( + dequant_fp8_block, + load_layer, + load_layer_dense, + resolve_checkpoint, +) +from models.demos.deepseek_v3_d_p.utils.kv_cache_utils import MlaKvCacheFormat +from tests.ttnn.utils_for_testing import comp_pcc + +SCHEDULES = {"sharing": (0, 2, 3, 20, 21, 24), "engram": (0, 1, 2, 3), "dspark": (20, 36, 37, 38, 39)} +SEQ = 512 +SCORED = 256 # scored prompt positions (the last ones) +# floor noise (user decisions 2026-09-29; playbook ~/knowledge/wiki/playbooks/Acceptance bars.md; evidence +# evidence/F9-drift-diagnosis): on every attention and MoE output, the relative error a component PCC bar of +# 0.999 allows (sqrt(2 * (1 - 0.999)) = 0.045; device components measure 3-5%); on every attention and MoE +# input, +# the level at which the reference's component matches the device's on oracle inputs (selection and routing +# flips): layer-20 attention 0.993 text / 0.988 random at 0.4% (small), real layer-20 MoE 0.9977 vs 0.9974 at +# 0.5% +NOISE = (0.045, 4e-3) +NOISE_SEEDS = 5 +# below the worst reference self-agreement: ~1 binomial SD of an agreement rate at 256 positions; PCC is +# smooth +MARGIN = {"top1": 0.02, "top5": 0.02, "pcc": 0.002} # token metrics: reported against the floor, not gated +# per-layer stream PCC below the reference's worst noisy drift (user decision 2026-09-29: gate per-layer +# drift; +# Gaussian noise matches the stream error's size but not its flip-shaped per-row distribution, which token +# agreement is sensitive to) +DRIFT_MARGIN = 0.003 +PRODUCTION_SEQ = 2048 +PRODUCTION_CANDIDATE_BLOCKS = 96 # of 128 visible blocks at S=2048 (2048 would make every block a candidate) +LONG_CHUNK = 5120 +# case -> (prompt tokens, chunk, candidate blocks; None = released 2048). Long cases: real weights only. +# padded_8120: a full chunk and a partial one (3000 of 5120 rows; graph.md rule 8) at production dims. +PRODUCTION_CASES = { + "one_chunk": (PRODUCTION_SEQ, PRODUCTION_SEQ, PRODUCTION_CANDIDATE_BLOCKS), + "two_chunks": (PRODUCTION_SEQ, PRODUCTION_SEQ // 2, PRODUCTION_CANDIDATE_BLOCKS), + **{f"{n}x{LONG_CHUNK}": (n * LONG_CHUNK, LONG_CHUNK, None) for n in (2, 4, 11)}, + "padded_8120": (LONG_CHUNK + 3000, LONG_CHUNK, None), + "4x5120_scaled_fp8": (4 * LONG_CHUNK, LONG_CHUNK, None), + # the 10,240-row chunk (TT_V41_CHUNK=10240) on the 2x5120 and 11x5120 + # prompts: one full chunk; five full chunks and a partial one (5,120 of 10,240 rows, the same last-chunk + # window + # [51200, 56320) as 11x5120). The same oracle, drifts and bars as the 5,120-row cases (the drift's "chunk" + # window stays the last LONG_CHUNK rows, so the cached references serve). + "1x10240": (2 * LONG_CHUNK, 2 * LONG_CHUNK, None), + "6x10240": (11 * LONG_CHUNK, 2 * LONG_CHUNK, None), +} +# cases whose compressed KV uses another format (test_block_v41 KV_FORMATS): the transformer's free-running +# prefill +# has no format choice, so these gate only the teacher-forced block acceptance at the last chunk, in that +# format +KV_FORMAT_CASES = {"4x5120_scaled_fp8": "scaled_fp8"} +MESH = [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) +] +MESH_4X2 = [ # LoudBox 4x2 (SP4 x TP2): production gate only (beads 8y7.20.*.2) + pytest.param( + (4, 2), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(4, 2), topology="mesh-4x2"), + id="fabric2d-mesh-4x2", + ) +] + + +def last_chunk_rows(case: str) -> int | None: + """The valid rows of the last chunk of a long production ``case`` (the rows its ``chunk`` gate window + covers); + None for the short cases (no chunk window).""" + seq, chunk, _ = PRODUCTION_CASES[case] + # a chunk that is a multiple of LONG_CHUNK keeps LONG_CHUNK's window: the cached drifts are per window + return seq - (seq - 1) // LONG_CHUNK * LONG_CHUNK if chunk % LONG_CHUNK == 0 else None + + +def setup_small(mesh_device, case, schedule): + """Everything before the small prefill: prompt, reference, Engram / DSpark modules and the transformer + (device + MoE tensors from / into the weight cache). Shared by the test and the cache prepare step (mock mesh).""" + layers = SCHEDULES[schedule] + chunk, total = {"one_chunk": (SEQ, SEQ), "two_chunks": (SEQ // 2, SEQ), "padded": (SEQ // 2, SEQ - 12)}[case] + spec = small_spec(layers, SEQ, dspark=schedule == "dspark") + tokens = orc.text_tokens(total) + reference = orc.LazyReference(spec)() # always needed here (Engram tables / DSpark weights); a phase event + orc.load_engram_rows(reference, spec, tokens) # synthetic Engram rows the prompt hashes to + engram, engram_hash = {}, None + if reference.engram_hash is not None: + engram_hash = V41EngramHash(SmallV41Config, reference.engram_hash.token_map) + for pos, layer in enumerate(layers): + e = reference.layers[pos].engram + if e is None: + continue + weights = { + "wkv": dequant_fp8_block(e.wkv.weight.detach(), e.wkv.scale.detach()), + "q_weight": e.q_weight.detach(), + "k_weight": e.k_weight.detach(), + } + table = V41EngramTable(e.embed.weight.detach(), e.embed.scale.detach(), e.embed.oracle_rows) + engram[layer] = TtV41Engram(mesh_device, SmallV41Config, layer, weights, table) + dspark = None + if reference.mtp: + fp8 = lambda linear: dequant_fp8_block(linear.weight.detach(), linear.scale.detach()) + dspark = { + "main_proj": fp8(reference.mtp[0].main_proj), + "main_norm": reference.mtp[0].main_norm.weight.detach(), + "layers": [{"wkv": fp8(b.attn.wkv), "kv_norm": b.attn.kv_norm.weight.detach()} for b in reference.mtp], + } + # device MoE tensors converted once per weight identity (tests/v41/weight_cache.py) + with _stage(f"{schedule} {case} build", "weights"): + model = TtV41Transformer( + mesh_device, + SmallV41Config, + list(layers), + lambda layer, include_moe: device_weights(reference, layers.index(layer), include_moe), + reference.embed.weight.detach(), + reference.norm.weight.detach(), + reference.head.weight.detach(), + max_seq_len=SEQ, + chunk=chunk, + dspark_weights=dspark, + engram=engram, + engram_hash=engram_hash, + weight_cache_path=weight_cache_dir(spec, mesh_device.shape), + ) + return model, spec, tokens, reference, dspark + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("schedule", list(SCHEDULES)) +@pytest.mark.parametrize("case", ["one_chunk", "two_chunks", "padded"]) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_transformer_small(mesh_device, device_params, case, schedule): + model, spec, tokens, reference, dspark = setup_small(mesh_device, case, schedule) + state, _ = _check(model, spec, tokens, reference, f"{schedule} {case}") + if dspark is not None: + result = orc.oracle(spec, tokens, model=reference) + # rings seeded from free-running device streams inherit their drift; reported only. The G1 ring bar + # (>= 0.998) applies with teacher-forced taps (user decision 2026-09-29), gated in test_dspark. + assert len(state.dspark_rings) == len(reference.mtp) + for k, ring in enumerate(state.dspark_rings): + device = ttnn.to_torch(ttnn.get_device_tensors(ring)[0])[0, 0] + pcc = comp_pcc(result["state"]["dspark_window"][k].float(), device.float(), 0.0)[1] + logger.info(f"transformer {schedule} {case}: DSpark ring {k} PCC {pcc:.5f} (free-running, reported)") + + +@contextmanager +def _stage(name: str, kind: str): + """Log a stage's start and elapsed time (visible while the run runs) and emit it as a ``kind`` timing + phase + (reference / oracle / weights / compute) for the lock-phase breakdown.""" + logger.info(f"stage {name}: start") + t = time.perf_counter() + with timing_events.phase(kind, name=name): + yield + logger.info(f"stage {name}: {time.perf_counter() - t:.1f}s") + + +def _agreement(expected: torch.Tensor, actual: torch.Tensor) -> dict: + """[positions, vocab] logits: top-1 agreement, top-5 recall of the expected top-1, logits PCC.""" + top1 = expected.argmax(-1) + return { + "top1": (actual.argmax(-1) == top1).float().mean().item(), + "top5": (actual.topk(5, dim=-1).indices == top1[:, None]).any(-1).float().mean().item(), + "pcc": comp_pcc(expected, actual, 0.0)[1], + } + + +def reference_data(spec, tokens, reference, last_chunk=None): + """What the gate compares against: the clean oracle, the scored tail logits, the per-layer noisy drifts + (with + ``last_chunk``, also over the last chunk's rows) and the token-agreement floor (all disk-cached; the + prepare step + fills them outside the device lock).""" + scored = min(SCORED, tokens.shape[1]) + clean = orc.oracle(spec, tokens, reference) + expected = orc.tail_logits(spec, tokens, scored, reference) + drifts = [ + orc.noise_drift(spec, tokens, scored, (*NOISE, s), reference, chunk=last_chunk) for s in range(NOISE_SEEDS) + ] + token_floor = {k: 1.0 for k in MARGIN} + for seed in range(NOISE_SEEDS): + noisy = _agreement(expected, orc.tail_logits(spec, tokens, scored, reference, noise=(*NOISE, seed))) + token_floor = {k: min(token_floor[k], noisy[k]) for k in token_floor} + return clean, expected, drifts, token_floor + + +def kv_format_reference(spec, tokens, reference): + """What a KV_FORMAT_CASES case compares against (disk-cached; the prepare step fills it): the clean oracle + and + the KV sources' unrounded compressed KV.""" + clean = orc.oracle(spec, tokens, reference) + return clean, unrounded_kv(spec, tokens, reference, clean) + + +def _check(model, spec, tokens, reference, name, last_chunk=None): + """Two prefills of ``tokens`` [1, S]: bit-identical; each layer's free-running streams (all rows, the last + SCORED rows and, with ``last_chunk``, the last chunk's rows) no further from the reference than its drift + under + the floor noise (worst seed) minus DRIFT_MARGIN. Token agreement vs the same floor is reported. Returns + the + first prefill's state and the clean oracle result.""" + scored = min(SCORED, tokens.shape[1]) + mesh, tp = model.mesh_device, model.mesh_device.shape[1] + concat = ttnn.ConcatMesh2dToTensor(mesh, tuple(mesh.shape), dims=(2, 3)) + streams = {} + + def observe(layer, x, start, length): + rows = ttnn.to_torch(x, mesh_composer=concat)[0, 0, :length] + streams.setdefault(layer, []).append(_unpack(rows, model.config.HC_MULT, tp)) + + with _stage(f"{name} prefill (compile + run, observed)", "compute"): + logits, state = model.prefill(tokens[0], scored, observe) + with _stage(f"{name} prefill (repeat)", "compute"): + logits2, _ = model.prefill(tokens[0], scored) + with _stage(f"{name} reference (cached unless precomputed)", "oracle"): + clean, expected, drifts, token_floor = reference_data(spec, tokens, reference, last_chunk) + assert torch.equal(logits, logits2), "prefill is not bit-identical across repeats" + dump = os.environ.get("TT_V41_TRANSFORMER_DUMP") + if dump: # cross-process determinism: compare two runs' dumps with torch.equal + final = torch.cat(streams[list(streams)[-1]]) + digests = { + l: hashlib.sha256(torch.cat(p).contiguous().view(torch.uint8).numpy().tobytes()).hexdigest() + for l, p in streams.items() + } + torch.save({"logits": logits, "final_stream": final, "stream_sha256": digests}, dump) + logger.info( + f"transformer {name}: dumped logits {tuple(logits.shape)}, final stream {tuple(final.shape)} to {dump}" + ) + tokens_device = _agreement(expected, logits) + logger.info( + f"transformer {name} tokens (reported): " + + ", ".join(f"{k} {tokens_device[k]:.4f} (reference self {token_floor[k]:.4f})" for k in MARGIN) + ) + failures = [] + windows = [("all", slice(None)), ("tail", slice(-scored, None))] + if last_chunk: + windows.append(("chunk", slice(-last_chunk, None))) + for layer, parts in streams.items(): + device, ref = torch.cat(parts), clean["blocks"][layer]["x_out"] + for key, rows in windows: + value = comp_pcc(ref[rows].float(), device[rows].float(), 0.0)[1] + floor = min(d[layer][key] for d in drifts) + logger.info(f"transformer {name} layer {layer} {key}: device {value:.5f} (bar {floor - DRIFT_MARGIN:.5f})") + if value < floor - DRIFT_MARGIN: + failures.append((layer, key, value, floor)) + assert not failures, failures + return state, clean + + +def _teacher_forced_last_chunk(model, clean, total, name, kv_format=MlaKvCacheFormat.BF16_RM, unrounded=None): + """The block acceptance of test_block_v41 at the last chunk of a chunked prefill of ``total`` tokens: + every chunk + through every block on the oracle's inputs (streams and pre-mix) over a fresh state, so each block reads + caches + it wrote itself from teacher-forced inputs. Gates the last chunk's block outputs (>= BLOCK_PCC real), the + KV + sources' compressed-KV and index-K rows (all rows) and every window carry (>= CACHE_PCC). With + ``kv_format`` + SCALED_FP8 the compressed KV is compared with ``unrounded`` (test_block_v41.unrounded_kv; vs the FP4-QDQ + rows + reported).""" + mesh, cfg, chunk = model.mesh_device, model.config, model.chunk + shape, tp, n = tuple(mesh.shape), mesh.shape[1], cfg.HC_MULT + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh, shape, dims=(2, 3))) + if kv_format == MlaKvCacheFormat.BF16_RM: + state = model._new_state() + else: + state = V41PrefillState(mesh, cfg, model.max_seq_len, chunk, model.layers, kv_format=kv_format) + last = {} + for start in range(0, total, chunk): + length = min(chunk, total - start) + for layer, block in zip(model.layers, model.blocks): + rec = clean["blocks"][layer] + x_rows = torch.zeros(chunk, *rec["x_in"].shape[1:]) + x_rows[:length] = rec["x_in"][start : start + length].float() + pre_rows = torch.zeros(chunk, rec["pre_in"].shape[-1]) + pre_rows[:length] = rec["pre_in"][start : start + length].float() + x = ttnn.from_torch( + _pack(x_rows, tp), + device=mesh, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh, shape, dims=(2, 3)), + ) + pre = ttnn.from_torch( + pre_rows[None, None], + device=mesh, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh, shape, dims=(2, None)), + ) + x_out, pre_out = block(x, pre, state, length) + if start + length == total: + last[layer] = (_unpack(down(x_out)[0, 0], n, tp)[:length], down(pre_out)[0, 0, :length, :n]) + for t in (x, pre, x_out, pre_out): + ttnn.deallocate(t) + state.advance(length) + first, failures = total - length, [] + for layer, (x_out, pre_out) in last.items(): + rec = clean["blocks"][layer] + entry = { + "block_out": (_pcc(rec["x_out"][first:total], x_out), BLOCK_PCC["real"]), + "pre_mix": (_pcc(rec["pre_out"][first:total], pre_out), None), + } + tail = rec["window_kv"][max(0, total - WINDOW_SLOT) : total].float() + carry = torch.zeros(WINDOW_SLOT, tail.shape[-1]) + carry[WINDOW_SLOT - tail.shape[0] :] = tail + entry["window_kv"] = (_pcc(carry, state.to_host(state.window_carry[layer])), CACHE_PCC) + if layer in C.KV_SOURCE_LAYERS: + pub = clean["shared"][layer] + rows, w0 = pub["compress_kv"].shape[0], state.geometry.window_rows + stored = state.to_host(state.kv[layer])[w0 : w0 + rows] + if kv_format == MlaKvCacheFormat.SCALED_FP8: + entry["compressed_kv"] = (_pcc(unrounded[layer], stored), CACHE_PCC) + entry["compressed_kv_vs_fp4"] = (_pcc(pub["compress_kv"], stored), None) + else: + entry["compressed_kv"] = (_pcc(pub["compress_kv"], stored), CACHE_PCC) + entry["index_k"] = (_pcc(pub["index_k"], state.to_host(state.index_k[layer])[:rows]), CACHE_PCC) + logger.info( + f"transformer {name} teacher-forced last chunk [{first}, {total}) layer {layer}: " + + ", ".join(f"{k} {v:.5f}" + (f" (bar {bar})" if bar else "") for k, (v, bar) in entry.items()) + ) + failures += [(layer, k, v, bar) for k, (v, bar) in entry.items() if bar is not None and v < bar] + assert not failures, failures + + +def setup_production(mesh_device, weights, case): + """Everything before the production prefill of ``case`` (PRODUCTION_CASES; see setup_small); None when the + checkpoint is not downloaded.""" + layers = SCHEDULES["sharing"] + seq, chunk, candidate_blocks = PRODUCTION_CASES[case] + ckpt = resolve_checkpoint() if weights == "real" else None + if weights == "real" and ckpt is None: + return None + spec = orc.real_spec( + layers, + seq, + candidate_topk_blocks=candidate_blocks, + checkpoint=ckpt.root if ckpt else None, + ) + cfg = C if candidate_blocks is None else type("V41TestConfig", (C,), {"CANDIDATE_TOPK_BLOCKS": candidate_blocks}) + tokens = orc.text_tokens(seq) + # built only on an oracle or weight-cache miss (warm runs construct no reference model) + reference = orc.LazyReference(spec) + root = weight_cache_dir(spec, mesh_device.shape) + if ckpt is None: + + def layer_weights(layer, include_moe): + pos, dense = layers.index(layer), f"layer_{layer}.dense" + if not include_moe: + return host_weights(root, dense, lambda: device_weights(reference(), pos, include_moe=False)) + weights = device_weights(reference(), pos) + host_weights(root, dense, lambda: {k: v for k, v in weights.items() if k not in MOE_KEYS}) + return weights + + top = host_weights( + root, + "top", + lambda: {k: getattr(reference(), k).weight.detach() for k in ("embed", "norm", "head")}, + ) + embed, norm, head = top["embed"], top["norm"], top["head"] + else: + layer_weights = lambda layer, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, layer) + top = ckpt.read(["embed.weight", "norm.weight", "head.weight"]) + embed, norm, head = top["embed.weight"], top["norm.weight"], top["head.weight"] + with _stage(f"production {weights} build (MoE weights cached after the first build)", "weights"): + model = TtV41Transformer( + mesh_device, + cfg, + list(layers), + layer_weights, + embed, + norm, + head, + max_seq_len=-(-seq // chunk) * chunk, # whole chunks; a padded last chunk writes nothing past seq + chunk=chunk, + weight_cache_path=root, + ) + return model, spec, tokens, reference + + +@pytest.mark.timeout(7200) +@pytest.mark.parametrize( + "weights, chunks", + [(w, c) for c in ("one_chunk", "two_chunks") for w in ("synthetic", "real")] + + [("real", c) for c in PRODUCTION_CASES if c not in ("one_chunk", "two_chunks")], + ids=lambda v: v, +) +@pytest.mark.parametrize("mesh_device, device_params", MESH + MESH_4X2 + galaxy_meshes(), indirect=True) +def test_v41_transformer_production(mesh_device, device_params, weights, chunks): + """Real dims, layers 0 2 3 20 21 24 (every sharing role and SWA-only; Engram layer 1 needs checkpoint + tables, not downloaded). Precompute the reference outside the device lock first + (tests/v41/prepare_caches.py: + oracle, ``tail_logits`` clean and per noise seed; disk-cached). Long cases (``x5120``) also gate the + last + chunk's rows and run the block acceptance at the last chunk (module docstring). Galaxy 8x4 / 4x8 (bead + 8y7.13.4): the same cases and bars; the oracle is mesh-independent (LoudBox caches are reused).""" + setup = setup_production(mesh_device, weights, chunks) + if setup is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + model, spec, tokens, reference = setup + seq, _, _ = PRODUCTION_CASES[chunks] + last = last_chunk_rows(chunks) + name = f"production {weights} chunks={chunks}" + if chunks in KV_FORMAT_CASES: + with _stage(f"{name} reference (cached unless precomputed)", "oracle"): + clean, unrounded = kv_format_reference(spec, tokens, reference) + with _stage(f"{name} teacher-forced blocks", "compute"): + _teacher_forced_last_chunk(model, clean, seq, name, KV_FORMATS[KV_FORMAT_CASES[chunks]], unrounded) + return + _, clean = _check(model, spec, tokens, reference, name, last_chunk=last) + if last: + with _stage(f"{name} teacher-forced blocks", "compute"): + _teacher_forced_last_chunk(model, clean, seq, name) + logger.info(f"{name}: reference model built: {reference.built}") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_transformer_vl_v41.py b/models/demos/deepseek_v3_d_p/tests/v41/test_transformer_vl_v41.py new file mode 100644 index 000000000000..9f1b143052e8 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_transformer_vl_v41.py @@ -0,0 +1,217 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""DeepSeek-V4.1 prefill of merged (image + text) prompts (bead 10.2): image span merge, VL routing bias, Engram mask. + +Acceptance: merged-sequence blocks meet the text block acceptance, so the gate is ``test_transformer_v41._check`` +unchanged (each layer's free-running streams, all rows and the scored text tail, vs the reference's worst noisy drift +minus DRIFT_MARGIN; bit-identical repeats; token agreement reported), on prompts whose first chunk holds image spans. +The reference is ``oracle.build_vl_reference``: the text reference plus each layer's ``bias_vl`` and the span +delimiters, run through ``Transformer.forward(images, token_types)``. Image features are teacher-forced (the same +aligner rows go to the reference and the device), so the gate measures the backbone on the merged sequence; the +encoder has its own bar (10.1) and the end-to-end image path is 10.3. The merged embedding itself is checked +bit-exact against the reference's (the merge copies rows). + +Cases: small dims, two images (sharing roles; Engram layers 0 1 2 3), one chunk and two chunks; production shape +(S=2048, one 640x480 image, 206 span positions) with synthetic and real weights (real: the checkpoint's aligner rows +of the image). Precompute the CPU oracles first: ``scripts/precompute_vl_oracles.py`` in the bead's artifacts. +""" + + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41 import vl_prompts +from models.demos.deepseek_v3_d_p.tests.v41.reference_weights import device_weights +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config, small_spec +from models.demos.deepseek_v3_d_p.tests.v41.test_transformer_v41 import ( + MESH, + NOISE, + NOISE_SEEDS, + PRODUCTION_CANDIDATE_BLOCKS, + SCHEDULES, + SCORED, + _check, + _stage, +) +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.engram import TtV41Engram, V41EngramHash, V41EngramTable +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer +from models.demos.deepseek_v3_d_p.tt.v41.weights import ( + dequant_fp8_block, + load_layer, + load_layer_dense, + resolve_checkpoint, +) + +DELIMITERS = ("image_start", "image_end", "image_newline") + + +class _MergedPrompt: + """A ``TtV41Transformer`` whose ``prefill`` carries one merged prompt's image inputs, so ``_check`` drives it.""" + + def __init__(self, model: TtV41Transformer, prompt: orc.VLPrompt): + self.model, self.mesh_device, self.config = model, model.mesh_device, model.config + self.token_types = prompt.token_types + replicate = ttnn.ReplicateTensorToMesh(model.mesh_device) + self.features = [ + ttnn.from_torch( + f[None, None], + device=model.mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=replicate, + ) + for f in prompt.features + ] + + def prefill(self, tokens, logit_positions=1, on_block=None): + return self.model.prefill(tokens, logit_positions, on_block, self.token_types, self.features) + + def merged_embedding(self, tokens: torch.Tensor) -> torch.Tensor: + """The first chunk's embedding after the image merge, [chunk, hidden] bf16 on host.""" + m, chunk = self.model, self.model.chunk + ids = torch.zeros(chunk, dtype=torch.int64) + ids[: min(chunk, tokens.numel())] = tokens[:chunk] + types = torch.full((chunk,), -1, dtype=torch.int64) + types[: min(chunk, tokens.numel())] = self.token_types[:chunk] + rows = m._column(types != -1) + h = m._merge_images(m.embedding(m._token_ids(ids)), types, self.features, rows) + concat = ttnn.ConcatMesh2dToTensor(m.mesh_device, tuple(m.mesh_device.shape), dims=(2, 3)) + return ttnn.to_torch(h, mesh_composer=concat)[0, 0] + + +def _reference_unless_cached(spec, tokens: torch.Tensor, prompt: orc.VLPrompt): + """None when every oracle ``_check`` reads is on disk (a real-weight reference build takes minutes), else the VL + reference: the oracle functions would otherwise build the text reference, which ignores the images.""" + scored = min(SCORED, tokens.shape[1]) + paths = [orc.cache_path(spec, tokens), orc._noisy_path(spec, tokens, scored, None, "tail")] + for seed in range(NOISE_SEEDS): + paths += [orc._noisy_path(spec, tokens, scored, (*NOISE, seed), kind) for kind in ("tail", "drift")] + return None if all(p.is_file() for p in paths) else orc.build_vl_reference(spec, prompt) + + +def _check_merge(merged: _MergedPrompt, tokens: torch.Tensor, result: dict, layer: int, name: str): + """The merged embedding equals the reference's (its first block input, stream 0) bit for bit.""" + chunk = merged.model.chunk + device = merged.merged_embedding(tokens[0]) + expected = result["blocks"][layer]["x_in"][:chunk, 0] + assert torch.equal(device[: expected.shape[0]].to(expected.dtype), expected), f"{name}: merged embedding differs" + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("schedule", ["sharing", "engram"]) +@pytest.mark.parametrize("case", ["one_chunk", "two_chunks"]) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_transformer_vl_small(mesh_device, device_params, case, schedule): + layers = SCHEDULES[schedule] + seq = vl_prompts.SMALL_SEQ + chunk = {"one_chunk": seq, "two_chunks": seq // 2}[case] + base = small_spec(layers, seq) + tokens, prompt = vl_prompts.small_prompt(SmallV41Config.EMB_SIZE) + spec = orc.vl_spec(base, prompt) + with _stage(f"vl {schedule} {case} reference", "reference"): + reference = orc.build_vl_reference(spec, prompt) + orc.load_engram_rows(reference, spec, tokens) # the rows of the masked hash (image tokens are DEAD) + engram, engram_hash = {}, None + if reference.engram_hash is not None: + engram_hash = V41EngramHash(SmallV41Config, reference.engram_hash.token_map) + for pos, layer in enumerate(layers): + e = reference.layers[pos].engram + if e is None: + continue + weights = { + "wkv": dequant_fp8_block(e.wkv.weight.detach(), e.wkv.scale.detach()), + "q_weight": e.q_weight.detach(), + "k_weight": e.k_weight.detach(), + } + table = V41EngramTable(e.embed.weight.detach(), e.embed.scale.detach(), e.embed.oracle_rows) + engram[layer] = TtV41Engram(mesh_device, SmallV41Config, layer, weights, table) + + def layer_weights(layer, include_moe): + pos = layers.index(layer) + weights = device_weights(reference, pos, include_moe) + return weights | {"gate_bias_vl": reference.layers[pos].ffn.gate.bias_vl.detach()} + + # the text weights are the text spec's: its device MoE tensors are reused + with _stage(f"vl {schedule} {case} build", "weights"): + model = TtV41Transformer( + mesh_device, + SmallV41Config, + list(layers), + layer_weights, + reference.embed.weight.detach(), + reference.norm.weight.detach(), + reference.head.weight.detach(), + max_seq_len=seq, + chunk=chunk, + engram=engram, + engram_hash=engram_hash, + image_embeds={k: getattr(reference, k).detach() for k in DELIMITERS}, + weight_cache_path=weight_cache_dir(base, mesh_device.shape), + ) + merged = _MergedPrompt(model, prompt) + with _stage(f"vl {schedule} {case} merge exactness", "compute"): + _check_merge(merged, tokens, orc.oracle(spec, tokens, reference), layers[0], f"vl {schedule} {case}") + _check(merged, spec, tokens, reference, f"vl {schedule} {case}") + + +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("weights", ["synthetic", "real"]) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_transformer_vl_production(mesh_device, device_params, weights): + """Real dims, layers 0 2 3 20 21 24, S=2048 in one chunk, one 640x480 image. Precompute the reference first.""" + layers = SCHEDULES["sharing"] + ckpt = resolve_checkpoint() if weights == "real" else None + if weights == "real" and ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + seq = vl_prompts.PRODUCTION_SEQ + base = orc.real_spec( + layers, seq, candidate_topk_blocks=PRODUCTION_CANDIDATE_BLOCKS, checkpoint=ckpt.root if ckpt else None + ) + cfg = type("V41TestConfig", (C,), {"CANDIDATE_TOPK_BLOCKS": PRODUCTION_CANDIDATE_BLOCKS}) + with _stage(f"vl production {weights} prompt (vision oracle cached)", "oracle"): + tokens, prompt, _ = vl_prompts.production_prompt(ckpt.root if ckpt else None) + spec = orc.vl_spec(base, prompt) + if ckpt is None: + reference = orc.build_vl_reference(spec, prompt) + + def layer_weights(layer, include_moe): + pos = layers.index(layer) + weights = device_weights(reference, pos, include_moe) + return weights | {"gate_bias_vl": reference.layers[pos].ffn.gate.bias_vl.detach()} + + embed, norm, head = ( + reference.embed.weight.detach(), + reference.norm.weight.detach(), + reference.head.weight.detach(), + ) + image_embeds = {k: getattr(reference, k).detach() for k in DELIMITERS} + else: + reference = _reference_unless_cached(spec, tokens, prompt) + layer_weights = lambda layer, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, layer) + top = ckpt.read(["embed.weight", "norm.weight", "head.weight", *DELIMITERS]) + embed, norm, head = top["embed.weight"], top["norm.weight"], top["head.weight"] + image_embeds = {k: top[k] for k in DELIMITERS} + with _stage(f"vl production {weights} build (MoE weights cached after the first build)", "weights"): + model = TtV41Transformer( + mesh_device, + cfg, + list(layers), + layer_weights, + embed, + norm, + head, + max_seq_len=seq, + chunk=seq, + image_embeds=image_embeds, + weight_cache_path=weight_cache_dir(base, mesh_device.shape), + ) + merged = _MergedPrompt(model, prompt) + with _stage(f"vl production {weights} merge exactness", "compute"): + _check_merge(merged, tokens, orc.oracle(spec, tokens, reference), layers[0], f"vl production {weights}") + _check(merged, spec, tokens, reference, f"vl production {weights}") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_all_to_all.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_all_to_all.py new file mode 100644 index 000000000000..907377c7b469 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_all_to_all.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Standalone ``all_to_all_async_generic`` at the V4.1 attention reshard shapes (bead 8y7.9.1). + +``TtV41Attention.forward`` reshards q head->sequence before ``sparse_sdpa`` (in the seq-major projection layout, heads +side by side: ``head_to_seq_flat``) and the output sequence->head after it, over TP (cluster_axis 1). On the LoudBox +2x4 mesh TP has 4 chips; on 4x2 it has 2. +Each case runs one direction on one mesh axis and checks the result bit-exact against the torch reshard: +global tensor sharded [in_dim over the axis, out_dim over the other axis] -> the same global tensor sharded +[out_dim over the axis, ...], i.e. the op must concatenate ``in_dim`` and split ``out_dim`` across the axis. + +Regression (4x2, cluster_axis 1; bead 8y7.13.7): the op's Fabric2D multicast initialization routes mesh axis 1 +east/west, and the LoudBox 4x2 TP axis is wired north-south; the writer stopped in ``fail_stop_invalid_fabric_route`` +and the collective hung. The host now enables that initialization only on axes wired in the kernel's direction. +``test_v41_tp_all_to_all`` covers the V4.1 reshard ``V41Collectives.tp_all_to_all`` that attention uses. +""" + +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes, on_galaxy +from models.demos.deepseek_v3_d_p.tt.tt_ccl import per_axis_topology +from models.demos.deepseek_v3_d_p.tt.v41.ccl import V41Collectives + +# per-chip q at the attention call site: [1, heads/tp, seq/sp, head_dim]; seq/sp of the whole chunk +SIZES = { + "small": dict(heads=32, seq=512, head_dim=128), # SmallV41Config, SMALL_SEQ + "production": dict(heads=64, seq=2048, head_dim=512), # V4.1 Flash, block test SEQ +} +DIRECTIONS = {"head_to_seq": (1, 2), "head_to_seq_flat": (3, 2), "seq_to_head": (2, 1)} # (in_dim, out_dim) +NUM_LINKS = (1, 2) + + +def _mesh(shape): + rows, cols = shape + return pytest.param( + shape, + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=shape, topology=f"mesh-{rows}x{cols}"), + id=f"fabric2d-mesh-{rows}x{cols}", + ) + + +def _make_input(mesh_device, cluster_axis, size, direction): + """Global torch tensor and its mesh tensor with the attention call site's per-chip shape.""" + dims = SIZES[size] + tp, sp = mesh_device.shape[1], mesh_device.shape[0] + axis_size, other_size = mesh_device.shape[cluster_axis], mesh_device.shape[1 - cluster_axis] + in_dim, out_dim = DIRECTIONS[direction] + # per-chip input of the attention reshard (head->seq: [1, H/tp, S/sp, D]; seq->head: [1, H, S/(sp*tp), D]) + if direction.startswith("head_to_seq"): + heads, rows = dims["heads"] // tp, dims["seq"] // sp + else: + heads, rows = dims["heads"], dims["seq"] // (sp * tp) + local = [1, heads, rows, dims["head_dim"]] + if direction == "head_to_seq_flat": # wq_b's output: [1, 1, S/sp, H/tp * D] + local = [1, 1, rows, dims["heads"] // tp * dims["head_dim"]] + global_shape = list(local) + global_shape[in_dim] *= axis_size + global_shape[out_dim] *= other_size + torch.manual_seed(0) + host = torch.randn(global_shape).to(torch.bfloat16) + placements = [None, None] + placements[cluster_axis] = ttnn.PlacementShard(in_dim) + placements[1 - cluster_axis] = ttnn.PlacementShard(out_dim) + mapper = ttnn.create_mesh_mapper(mesh_device, ttnn.MeshMapperConfig(placements, mesh_device.shape)) + x = ttnn.from_torch( + host, + device=mesh_device, + layout=ttnn.TILE_LAYOUT, + dtype=ttnn.bfloat16, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=mapper, + ) + assert list(ttnn.get_device_tensors(x)[0].shape) == local + return host, x, local + + +def _check(mesh_device, y, host, local, cluster_axis, direction): + """Device (axis a, other o) must hold the full ``in_dim`` and global ``out_dim`` block o * axis_size + a.""" + in_dim, out_dim = DIRECTIONS[direction] + axis_size = mesh_device.shape[cluster_axis] + expect_local = list(local) + expect_local[in_dim] *= axis_size + expect_local[out_dim] //= axis_size + block = expect_local[out_dim] + for idx, shard in enumerate(ttnn.get_device_tensors(y)): + coord = divmod(idx, mesh_device.shape[1]) + a, o = coord[cluster_axis], coord[1 - cluster_axis] + got = ttnn.to_torch(shard) + assert list(got.shape) == expect_local, f"device {coord}: shape {list(got.shape)} != {expect_local}" + want = host.narrow(out_dim, (o * axis_size + a) * block, block) + assert torch.equal(got, want), f"device {coord}: reshard mismatch" + + +@pytest.mark.timeout(600) +@pytest.mark.parametrize("num_links", NUM_LINKS, ids=[f"links{n}" for n in NUM_LINKS]) +@pytest.mark.parametrize("direction", list(DIRECTIONS)) +@pytest.mark.parametrize("size", list(SIZES)) +@pytest.mark.parametrize("cluster_axis", [1, 0], ids=["tp_axis", "sp_axis"]) +@pytest.mark.parametrize("mesh_device, device_params", [_mesh((2, 4)), _mesh((4, 2))] + galaxy_meshes(), indirect=True) +@pytest.mark.uncollect_if(pred=lambda mesh_device, size, **_: on_galaxy(mesh_device) and size == "small") +def test_v41_all_to_all(mesh_device, device_params, cluster_axis, size, direction, num_links): + """The shared op itself, on both mesh axes of both meshes (4x2 tp_axis is the hang regression).""" + t0 = time.time() + host, x, local = _make_input(mesh_device, cluster_axis, size, direction) + topology = per_axis_topology()[cluster_axis] + logger.info( + f"a2a {tuple(mesh_device.shape)} axis={cluster_axis} ({mesh_device.shape[cluster_axis]} chips) {direction} " + f"local={local} links={num_links} topology={topology} input ready {time.time() - t0:.1f}s" + ) + in_dim, out_dim = DIRECTIONS[direction] + t1 = time.time() + y = ttnn.experimental.all_to_all_async_generic( + x, + in_dim=in_dim, + out_dim=out_dim, + num_links=num_links, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + topology=topology, + cluster_axis=cluster_axis, + ) + ttnn.synchronize_device(mesh_device) + logger.info(f"a2a device done {time.time() - t1:.2f}s") + _check(mesh_device, y, host, local, cluster_axis, direction) + logger.info(f"a2a PASS total {time.time() - t0:.1f}s") + + +@pytest.mark.timeout(600) +@pytest.mark.parametrize("direction", list(DIRECTIONS)) +@pytest.mark.parametrize("size", list(SIZES)) +@pytest.mark.parametrize("mesh_device, device_params", [_mesh((2, 4)), _mesh((4, 2))] + galaxy_meshes(), indirect=True) +@pytest.mark.uncollect_if(pred=lambda mesh_device, size, **_: on_galaxy(mesh_device) and size == "small") +def test_v41_tp_all_to_all(mesh_device, device_params, size, direction): + """``V41Collectives.tp_all_to_all`` (the attention reshard) is bit exact and repeatable on both meshes.""" + t0 = time.time() + host, x, local = _make_input(mesh_device, 1, size, direction) + ccl = V41Collectives(mesh_device) + in_dim, out_dim = DIRECTIONS[direction] + outputs = [] + for rep in range(2): # the second call reuses the program cache and cycles the CCL semaphores + t1 = time.time() + outputs.append(ccl.tp_all_to_all(x, in_dim, out_dim)) + ttnn.synchronize_device(mesh_device) + logger.info( + f"tp_all_to_all {tuple(mesh_device.shape)} {direction} local={local} rep {rep} {time.time() - t1:.2f}s" + ) + for y in outputs: + _check(mesh_device, y, host, local, 1, direction) + logger.info(f"tp_all_to_all PASS total {time.time() - t0:.1f}s") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_attention_glue.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_attention_glue.py new file mode 100644 index 000000000000..e9cdb064d584 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_attention_glue.py @@ -0,0 +1,199 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""V4.1 attention glue around ``sparse_sdpa`` (bead 8y7.9.7): contracts of the reshaped data paths. + +* the fused head layout through the TP reshards (bead 8y7.9.9) equals the previous path bit for bit: q as + ``wq_b`` emits it, resharded head->sequence (``in_dim`` 3: the chips' heads side by side) and split into + sparse_sdpa's row-major heads with the RoPE tail by ``q_heads``, against nlp_create_qkv_heads + reshard (``in_dim`` + 1) + the RoPE tail glue (slice / rotary / untilize / slice_write); the sparse_sdpa output taken to the tiled head + groups with the inverse RoPE by ``o_heads`` and resharded sequence->head, against the inverse RoPE tail glue + + tilize + reshard + nlp_concat_heads of each group. +* ``_index_rows`` returns, per query, its valid KV rows in [window, top-k] order followed by a sentinel tail, on the + first chunk (queries with missing window rows: compacted) and on a later chunk (no compaction). The expected rows + are built in torch from the same window table and top-k. +* the grouped output projection's ``nlp_concat_heads`` of [1, groups, S, rank] equals the per-group concatenation. + +Shapes are the per-chip call-site shapes at chunk 5120 (production) and the small config. +""" + +from types import SimpleNamespace + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config +from models.demos.deepseek_v3_d_p.tt.mla.rope import get_rot_transformation_mat +from models.demos.deepseek_v3_d_p.tt.v41 import rope as v41_rope +from models.demos.deepseek_v3_d_p.tt.v41.attention import TOPK_ALIGN, TtV41Attention +from models.demos.deepseek_v3_d_p.tt.v41.cache import V41ChunkTables +from models.demos.deepseek_v3_d_p.tt.v41.ccl import V41Collectives +from models.demos.deepseek_v3_d_p.tt.v41.head_layout import o_heads, q_heads +from models.demos.deepseek_v3_d_p.tt.v41.layout import TP_AXIS + +MESH = [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) +] +# chunk and config per size: per chip q [1, heads/tp, chunk/sp, head_dim] +SIZES = {"small": (SmallV41Config, 512), "production": (C, 5120)} +SENTINEL = -1 + + +def _glue(mesh_device, cfg): + """The attributes the glue methods read, without building the attention weights.""" + trans = ttnn.from_torch( + get_rot_transformation_mat(), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + return SimpleNamespace(rope_dim=cfg.QK_ROPE_HEAD_DIM, trans_mat=trans, window=cfg.SLIDING_WINDOW) + + +def _mesh_tensor(mesh_device, t, dims, layout=ttnn.TILE_LAYOUT, dtype=ttnn.bfloat16): + shape = tuple(mesh_device.shape) + return ttnn.from_torch( + t, device=mesh_device, dtype=dtype, layout=layout, mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims) + ) + + +def _down(mesh_device, t): + shape = tuple(mesh_device.shape) + return ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(0, 1))) + + +def _cos_sin(mesh_device, cfg, chunk): + """Per-chip cos / sin of the chip's SP rows (replicated over TP), as ``V41ChunkTables.rope``.""" + sp = mesh_device.shape[0] + cos, sin = v41_rope.cos_sin(cfg, True, torch.arange(chunk)) + return tuple(_mesh_tensor(mesh_device, t.reshape(sp, 1, chunk // sp, -1), (0, None)) for t in (cos, sin)) + + +def _old_rope_tail(g, t, cos, sin): + """``TtV41Attention._rope_tail`` before 8y7.9.9: tiled [b, h, s, d] -> row-major, RoPE on the tail.""" + b, h, s, d = t.shape + tail = ttnn.slice(t, [0, 0, 0, d - g.rope_dim], [b, h, s, d]) + tail = ttnn.experimental.rotary_embedding_llama(tail, cos, sin, g.trans_mat, is_decode_mode=False) + out = ttnn.to_layout(t, ttnn.ROW_MAJOR_LAYOUT) + ttnn.experimental.slice_write( + ttnn.to_layout(tail, ttnn.ROW_MAJOR_LAYOUT), out, [0, 0, 0, d - g.rope_dim], [b, h, s, d], [1, 1, 1, 1] + ) + return out + + +def _old_inverse_rope_tail(g, t, cos, sin): + """``TtV41Attention._inverse_rope_tail`` before 8y7.9.9: row-major [b, h, s, d] (in place) -> tiled.""" + b, h, s, d = t.shape + tail = ttnn.to_layout(ttnn.slice(t, [0, 0, 0, d - g.rope_dim], [b, h, s, d]), ttnn.TILE_LAYOUT) + tail = ttnn.experimental.rotary_embedding_llama(tail, cos, ttnn.neg(sin), g.trans_mat, is_decode_mode=False) + ttnn.experimental.slice_write( + ttnn.to_layout(tail, ttnn.ROW_MAJOR_LAYOUT), t, [0, 0, 0, d - g.rope_dim], [b, h, s, d], [1, 1, 1, 1] + ) + return ttnn.to_layout(t, ttnn.TILE_LAYOUT) + + +@pytest.mark.parametrize("size", list(SIZES)) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_head_layout_matches_glue(mesh_device, device_params, size): + cfg, chunk = SIZES[size] + sp, tp = tuple(mesh_device.shape) + g, ccl = _glue(mesh_device, cfg), V41Collectives(mesh_device) + heads, d, groups = cfg.NUM_ATTENTION_HEADS, cfg.HEAD_DIM, cfg.O_GROUPS + cos, sin = _cos_sin(mesh_device, cfg, chunk) + cos_s, sin_s = (ttnn.mesh_partition(t, dim=2, cluster_axis=TP_AXIS) for t in (cos, sin)) + torch.manual_seed(0) + # q per chip as wq_b emits it: [1, 1, S/sp, H/tp * d] + q_host = torch.randn(sp, 1, chunk // sp, heads * d).to(torch.bfloat16) + q = _mesh_tensor(mesh_device, q_host, (0, 3)) + old_q, _, _ = ttnn.experimental.nlp_create_qkv_heads( + q, num_heads=heads // tp, num_kv_heads=0, transpose_k_heads=False + ) + old = _old_rope_tail(g, ccl.tp_all_to_all(old_q, in_dim=1, out_dim=2), cos_s, sin_s) + new = q_heads(ccl.tp_all_to_all(q, in_dim=3, out_dim=2), cos_s, sin_s, g.trans_mat, heads, g.rope_dim) + assert new.layout == ttnn.ROW_MAJOR_LAYOUT and tuple(new.shape) == tuple(old.shape), (new.shape, old.shape) + assert torch.equal(_down(mesh_device, old), _down(mesh_device, new)) + + # sparse_sdpa output: per chip [1, H, S/(sp*tp), d] row-major -> per chip [1, G/tp, S/sp, (H/G) * d] tiled + o_host = torch.randn(sp, tp * heads, chunk // (sp * tp), d).to(torch.bfloat16) + o_old = _mesh_tensor(mesh_device, o_host, (0, 1), layout=ttnn.ROW_MAJOR_LAYOUT) + o_new = _mesh_tensor(mesh_device, o_host, (0, 1), layout=ttnn.ROW_MAJOR_LAYOUT) + old = ccl.tp_all_to_all(_old_inverse_rope_tail(g, o_old, cos_s, sin_s), in_dim=2, out_dim=1) + groups_local, rows = groups // tp, chunk // sp + old = ttnn.reshape(old, [groups_local, heads // groups, rows, d]) + old = ttnn.reshape(ttnn.experimental.nlp_concat_heads(old), [1, groups_local, rows, heads // groups * d]) + new = o_heads(o_new, cos_s, ttnn.neg(sin_s), g.trans_mat, groups, g.rope_dim) + new = ccl.tp_all_to_all(new, in_dim=2, out_dim=1) + assert tuple(new.shape) == tuple(old.shape), (new.shape, old.shape) + assert torch.equal(_down(mesh_device, old), _down(mesh_device, new)) + + +def _expected_rows(window: torch.Tensor, comp: torch.Tensor | None, base: int, k: int) -> torch.Tensor: + """[q, W] window rows (-1: none) and [q, t] top-k (-1 tail) -> [q, k]: valid window rows, valid top-k + base, -1.""" + out = torch.full((window.shape[0], k), SENTINEL, dtype=torch.int64) + for i in range(window.shape[0]): + valid = [int(v) for v in window[i] if v != SENTINEL] + if comp is not None: + valid += [int(v) + base for v in comp[i] if v != SENTINEL] + out[i, : len(valid)] = torch.tensor(valid, dtype=torch.int64) + return out + + +@pytest.mark.parametrize("start", [0, 5120], ids=["first_chunk", "later_chunk"]) +@pytest.mark.parametrize("compressed", [False, True], ids=["window_only", "window_topk"]) +@pytest.mark.parametrize("size", list(SIZES)) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_index_rows_valid_first(mesh_device, device_params, size, compressed, start): + cfg, chunk = SIZES[size] + sp, tp = tuple(mesh_device.shape) + q_rows = chunk // (sp * tp) + start = start * chunk // 5120 # a later chunk: the second one + tables = V41ChunkTables(mesh_device, cfg, 2 * chunk, chunk, [2]) + window_width = cfg.SLIDING_WINDOW + width = -(-(window_width + (cfg.INDEX_TOPK if compressed else 0)) // TOPK_ALIGN) * TOPK_ALIGN + g = _glue(mesh_device, cfg) + g.compact_rows = g.window - 1 + window = tables.window_rows(start) + window_host = _down(mesh_device, window).reshape(-1, window_width) # chip order = query order + comp, comp_host, base = None, None, 128 + chunk + if compressed: + # top-k per query: distinct rows of the visible ones, sentinel tail (a varying number of valid slots) + torch.manual_seed(1) + topk = cfg.INDEX_TOPK + comp_host = torch.full((sp * tp * q_rows, topk), SENTINEL, dtype=torch.int64) + for i in range(comp_host.shape[0]): + visible = (start + i + 1) // 2 + n = min(topk, visible) + comp_host[i, :n] = torch.randperm(max(visible, 1))[:n] + comp = _mesh_tensor( + mesh_device, comp_host.reshape(sp, tp, q_rows, topk).to(torch.int32), (0, 1), dtype=ttnn.int32 + ) + comp = ttnn.to_layout(ttnn.typecast(comp, ttnn.uint32), ttnn.ROW_MAJOR_LAYOUT) # as topk_large_indices + rows = TtV41Attention._index_rows(g, window, base, comp, start) + assert rows.layout == ttnn.ROW_MAJOR_LAYOUT and rows.dtype == ttnn.uint32 + got = _down(mesh_device, ttnn.typecast(ttnn.to_layout(rows, ttnn.TILE_LAYOUT), ttnn.int32)) + got = got.reshape(-1, width).to(torch.int64) + expected = _expected_rows(window_host, comp_host, base, width) + assert torch.equal(got, expected), (got[:4, :8], expected[:4, :8]) + + +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_grouped_concat_heads(mesh_device, device_params): + """wo_a output per chip [1, groups, S, rank] -> [1, 1, S, groups * rank] (groups side by side).""" + sp, tp = tuple(mesh_device.shape) + groups, rows, rank = C.O_GROUPS // tp, 5120 // sp, C.O_LORA_RANK + torch.manual_seed(2) + host = torch.randn(sp, tp, groups, rows, rank).to(torch.bfloat16) + t = _mesh_tensor(mesh_device, host.reshape(sp, tp * groups, rows, rank), (0, 1)) + out = _down(mesh_device, ttnn.experimental.nlp_concat_heads(t)) + expected = host.permute(0, 1, 3, 2, 4).reshape(sp, tp, rows, groups * rank) + assert torch.equal(out.reshape(sp, tp, rows, groups * rank), expected) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_attention_matmuls.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_attention_matmuls.py new file mode 100644 index 000000000000..236c610bf584 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_attention_matmuls.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Dense matmuls of the V4.1 attention on ``tt/v41/attention.dense_program_config`` (bead 8y7.9.7, G2 gap #10). + +Per chip on LoudBox 2x4 (rows = chunk / sp; tiles [M, K] x [K, N], M = 80 at chunk 5120, 32 at 2048): +wq_a [M, 40] x [40, 40], wkv [M, 40] x [40, 16], wq_b [M, 40] x [40, 256], wo_a batched 2 x [M, 128] x [128, 32]; bf16 +activations, bfp8 weights, HiFi2 (``DENSE_COMPUTE_CONFIG`` = ttnn's default for these inputs, checked bit-exact). +Each K block of ``IN0_BLOCK_W`` is timed by trace replay (``ITERS`` back-to-back calls per trace, min over replays of +the device-synchronized wall per call) and compared with the fp32 torch product of the same device inputs: bf16 +partial sums round more with wider K blocks. Contract: the attention's chosen K block (``DENSE_IN0_BLOCK_W``) is +faster than ttnn's default config and within ``ACCURACY_SLACK`` of its PCC. Rejected alternative (measured): q heads +straight from wq_b as a batched in0-reuse 1D matmul (16 x [80, 40] x [40, 16]) took 880-1600 us vs 382 + 198 us for +wq_b + nlp_create_qkv_heads. +""" + +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41.attention import DENSE_COMPUTE_CONFIG, DENSE_IN0_BLOCK_W, dense_program_config +from tests.ttnn.utils_for_testing import comp_pcc + +MESH = [ + pytest.param( + (2, 4), + fabric2d_device_params(trace_region_size=64 << 20), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) +] +ITERS = 10 +M = 2560 + + +# name -> (in0 shape per M, in1 shape): per chip, 2x4 mesh +CASES = { + "wq_a": (lambda m: (1, 1, m, 1280), (1, 1, 1280, 1280)), + "wkv": (lambda m: (1, 1, m, 1280), (1, 1, 1280, 512)), + "wq_b": (lambda m: (1, 1, m, 1280), (1, 1, 1280, 8192)), + "wo_a": (lambda m: (1, 2, m, 4096), (1, 2, 4096, 1024)), +} +ROWS = {"chunk5120": 2560, "chunk2048": 1024} # rows per chip (chunk / sp) +IN0_BLOCK_W = (2, 4, 5, 8, 16) +ACCURACY_SLACK = 2e-5 # PCC vs fp32 the chosen config may lose against the default + + +def _timed_us(mesh_device, fn, replays: int = 5) -> float: + """Min over ``replays`` trace replays of the per-call wall (``ITERS`` calls per trace).""" + fn() # compile + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + for _ in range(ITERS): + fn() + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=True) # warm + best = float("inf") + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=True) + best = min(best, (time.perf_counter() - t0) * 1e6 / ITERS) + ttnn.release_trace(mesh_device, tid) + return best + + +@pytest.mark.timeout(1200) +@pytest.mark.parametrize("rows", list(ROWS)) +@pytest.mark.parametrize("case", list(CASES)) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_attention_matmul_configs(mesh_device, device_params, case, rows): + a_shape, b_shape = CASES[case] + a_shape = a_shape(ROWS[rows]) + torch.manual_seed(0) + rep = ttnn.ReplicateTensorToMesh(mesh_device) + a = ttnn.from_torch( + torch.randn(a_shape).to(torch.bfloat16), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + b = ttnn.from_torch( + (torch.randn(b_shape) / b_shape[-2] ** 0.5).to(torch.bfloat16), + device=mesh_device, + dtype=ttnn.bfloat8_b, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=rep, + ) + first = lambda t: ttnn.to_torch(ttnn.get_device_tensors(t)[0]).float() + exact = torch.matmul(first(a), first(b)) + default = first(ttnn.linear(a, b)) + hifi2 = DENSE_COMPUTE_CONFIG + assert torch.equal(default, first(ttnn.linear(a, b, compute_kernel_config=hifi2))), "default is not HiFi2" + pcc = lambda x, y: comp_pcc(x, y, 0.0)[1] + results = {"default": (_timed_us(mesh_device, lambda: ttnn.linear(a, b)), pcc(exact, default))} + m, k, n = a_shape[-2], b_shape[-2], b_shape[-1] + for w in IN0_BLOCK_W: + cfg = dense_program_config(m, k, n, w) + if cfg is None: + continue + run = lambda cfg=cfg: ttnn.linear(a, b, program_config=cfg, compute_kernel_config=hifi2) + results[f"k{w}"] = (_timed_us(mesh_device, run), pcc(exact, first(run()))) + base, base_pcc = results["default"] + for name, (us, vs_exact) in sorted(results.items(), key=lambda kv: kv[1][0]): + logger.info(f"MATMUL {case} {rows} {name}: {us:.1f} us ({base / us:.2f}x) pcc vs fp32 {vs_exact:.6f}") + chosen = f"k{DENSE_IN0_BLOCK_W[case]}" + us, vs_exact = results[chosen] + assert vs_exact >= base_pcc - ACCURACY_SLACK, (chosen, results) + assert us < base, (chosen, results) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_batch_decode.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_batch_decode.py new file mode 100644 index 000000000000..424b0d4057e1 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_batch_decode.py @@ -0,0 +1,381 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""V4.1 batched decode on the Galaxy: every slot of the pool a different prompt, against the vendored CPU +reference. + +Prefills ``sp * V41_BATCH_USERS`` prompts of real text (different windows of the text, lengths of both +parities +around ``V41_BATCH_PROMPT``) with the layer schedule ``V41_BATCH_LAYERS`` on the device +(``TtV41Transformer.prefill``, +each joining a slot of ``TtV41BatchDecoder``), then decodes ``V41_BATCH_STEPS`` tokens for all of them in one +step +each, teacher-forced on the text's own next tokens. The slots in ``V41_BATCH_REF_SLOTS`` are also run on the +CPU +reference (``Transformer.forward``; prefill then the same tokens step by step), and per step report the logits +PCC, +the greedy agreement and every block's output PCC. With ``V41_BATCH_TRACE=1`` (default) the step is captured +once +after ``V41_BATCH_WARMUP`` eager steps and replayed; replays report the device time. ``V41_BATCH_REF=0`` runs +the +device alone (checkpoint Engram schedules), judged by ``book_top1`` over all slots. ``V41_BATCH_LIVE=k`` steps +``k`` slots a step instead of all of them, rotating over the slots (step ``i`` steps slots ``i k .. i k + k - +1`` +modulo the slot count; each slot its own token counter), the c1 / c2 shape of the served step (with +``V41_DECODE_C1_MOE=1`` and ``k = 1`` the c1 MoE path); ``V41_BATCH_LIVE_FROM=n`` steps every +slot for the first ``n`` steps (the served order: the 32-row variant captured first); no CPU reference in that +mode. +The pool's window ring is indexed by the global step (a user is assumed to step every step), so a rotating +slot's +window goes stale after ``RING`` steps and ``book_top1`` is meaningful only for a run of up to a few rotations +or +with ``V41_BATCH_LIVE_ROTATE=0`` (the same ``k`` slots every step, the served c1 shape). + +Results go to the log as ``V41_BATCH_RESULT {json}`` and to ``V41_BATCH_OUT`` when set. Reported, not gated +here: a NaN or a crash fails the test, a low PCC does not. +""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_decode import _pcc, _reference_forward +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_depth import _dram_bytes, _rss_gb, parse_layers +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.batch_decode import TtV41BatchDecoder +from models.demos.deepseek_v3_d_p.tt.v41.engram_checkpoint import build_engram_modules +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint + +try: + from tracy import signpost +except ModuleNotFoundError: # not under the tracy wrapper + + def signpost(_name: str) -> None: + pass + + +DEFAULT_LAYERS = "0,2,3,20,21,24" +DEFAULT_REF_SLOTS = "0,1,9,14,22,31" +TRACE_REGION = 512 * 1024 * 1024 + + +def _row_to_host(x: ttnn.Tensor, rank: int, u: int, tp: int, hc: int) -> torch.Tensor: + """Row ``u`` of the decode tile's streams on mesh row ``rank`` (its TP shards) -> ``[hc, hidden]``.""" + shards = ttnn.get_device_tensors(x) + parts = [ttnn.to_torch(shards[rank * tp + j])[0, 0, u].float().view(hc, -1) for j in range(tp)] + return torch.cat(parts, dim=-1) + + +def _debug_join(decoder, state, layers) -> dict: + """Slot 0's pool rows against the state's (device 0): max abs difference of every layer's ring and every + source's compressed rows and index keys.""" + pool, out = decoder.pool, {} + for layer in layers: + base = pool.ring_row(layer) + ring = state.to_host(pool.kv_tensor(layer))[base : base + 128] + carry = state.to_host(state.window_carry[layer]) + out[f"ring{layer}"] = round(float((ring - carry).abs().max()), 5) + g = state.geometry + for src in state.kv: + written = state.start // decoder.config.compress_ratio(src) + comp = state.to_host(pool.kv[src])[pool.comp_row(src) : pool.comp_row(src) + written] + rows = state.to_host(state.kv[src])[g.kv_row_of_compressed(0) : g.kv_row_of_compressed(0) + written] + out[f"comp{src}"] = round(float((comp - rows).abs().max()), 5) + keys = state.to_host(pool.index_k[src])[:written] # [0, 0]: slot 0's tiled rows, untilized on the host + out[f"keys{src}"] = round(float((keys - state.to_host(state.index_k[src])[:written]).abs().max()), 5) + if src in pool.index_k_blocks: + blocks = state.to_host(pool.index_k_blocks[src]).reshape(-1, keys.shape[1])[:written] + out[f"blocks{src}"] = round(float((blocks - state.to_host(state.index_k[src])[:written]).abs().max()), 5) + out[f"comp{src}_norm"] = round(float(rows.abs().max()), 3) + for src, (k0, s0) in pool.carry.items(): + carry = state.compressor_carry.get(src) + if carry is not None: + out[f"carry{src}"] = round(float((state.to_host(k0)[0] - state.to_host(carry[0])[0]).abs().max()), 5) + return out + + +def _release(state) -> None: + """Free a prefill state's device tensors (its rows live in the pool now).""" + for group in (state.kv, state.index_k, state.window_carry): + for t in group.values(): + ttnn.deallocate(t) + if state.swa_scratch is not None: + ttnn.deallocate(state.swa_scratch) + for carry in state.carry_buffers.values(): + for t in carry: + ttnn.deallocate(t) + + +@pytest.mark.timeout(int(os.environ.get("V41_BATCH_TIMEOUT", 4 * 3600))) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=TRACE_REGION), indirect=True) +def test_v41_batch_decode(mesh_device, device_params): + layers = parse_layers(os.environ.get("V41_BATCH_LAYERS", DEFAULT_LAYERS)) + base = int(os.environ.get("V41_BATCH_PROMPT", 255)) + steps = int(os.environ.get("V41_BATCH_STEPS", 16)) + chunk = int(os.environ.get("V41_BATCH_CHUNK", 1024)) + users = int(os.environ.get("V41_BATCH_USERS", 4)) + use_ref = os.environ.get("V41_BATCH_REF", "1") == "1" + trace = os.environ.get("V41_BATCH_TRACE", "1") == "1" + warmup = int(os.environ.get("V41_BATCH_WARMUP", 2)) + sample = os.environ.get("V41_BATCH_SAMPLE", "0") == "1" # argmax on device (DeviceSampler) instead of logits + live = int(os.environ.get("V41_BATCH_LIVE", 0)) # slots stepping per step (0: all) + live_from = int(os.environ.get("V41_BATCH_LIVE_FROM", 0)) # the steps before it step every slot + live_rotate = os.environ.get("V41_BATCH_LIVE_ROTATE", "1") == "1" # 0: the same first slots every step + profile = os.environ.get("V41_BATCH_PROFILE", "0") == "1" # as test_v41_decode: drain the device profiler per block + ref_slots = ( + [int(s) for s in os.environ.get("V41_BATCH_REF_SLOTS", DEFAULT_REF_SLOTS).split(",") if s] if use_ref else [] + ) + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + from transformers import AutoTokenizer + + sp, tp = tuple(mesh_device.shape) + n_slots = sp * users + ref_slots = [s for s in ref_slots if s < n_slots] + # prompt lengths of both parities around the base, each slot its own window of the text + lengths = [base - (s * 5) % 32 for s in range(n_slots)] + assert min(lengths) >= C.SLIDING_WINDOW + stride = base + steps + 8 + try: + text = orc.text_tokens(n_slots * stride)[0] + except ValueError as e: # the book is shorter than the slots' windows laid end to end: overlap them + have = int(str(e).split()[0]) + stride = max(C.SLIDING_WINDOW, (have - stride) // max(n_slots - 1, 1)) + text = orc.text_tokens((n_slots - 1) * stride + base + steps + 8)[0] + logger.info(f"batch decode: {have} tokens of text, the slots' windows overlap at a stride of {stride}") + prompts = {s: text[s * stride : s * stride + lengths[s]] for s in range(n_slots)} + books = {s: text[s * stride + lengths[s] : s * stride + lengths[s] + steps + 1] for s in range(n_slots)} + if os.environ.get("V41_BATCH_SAME_PROMPT", "0") == "1": # every slot the same request: identical expert routing + lengths = [lengths[0]] * n_slots + prompts = {s: prompts[0] for s in range(n_slots)} + books = {s: books[0] for s in range(n_slots)} + max_seq = -(-(base + steps + 32) // chunk) * chunk + result = { + "layers": layers, + "n_layers": len(layers), + "slots": n_slots, + "users_per_rank": users, + "lengths": lengths, + "steps": steps, + "chunk": chunk, + "max_seq": max_seq, + "reference_slots": ref_slots, + "trace": trace, + "warmup": warmup, + "sample_on_device": sample, + "live": live, + "mesh": list(mesh_device.shape), + "fabric": str(device_params.get("fabric_config")), + } + spec = orc.real_spec(tuple(layers), max_seq, checkpoint=ckpt.root) + root = weight_cache_dir(spec, mesh_device.shape) + tokenizer = AutoTokenizer.from_pretrained(orc.HF_SNAPSHOT) + + # --- the reference, slot by slot: prefill logits, then per step the logits and every block's output --- + reference = {} + if ref_slots: + t0 = time.time() + ref = orc.build_reference(spec) + result["reference_build_s"] = round(time.time() - t0, 1) + blocks = {} + hooks = [ + ref.layers[pos].register_forward_hook( + lambda mod, args, out, layer=layer: blocks.__setitem__(layer, out[0][0, 0].float()) + ) + for pos, layer in enumerate(layers) + ] + try: + t0 = time.time() + for s in ref_slots: + orc._reset_state(ref) + entry = {"prefill": _reference_forward(ref, prompts[s][None], 0), "steps": []} + for i in range(steps): + blocks.clear() + logits = _reference_forward(ref, torch.tensor([[int(books[s][i])]]), lengths[s] + i) + entry["steps"].append((logits, {layer: blocks[layer].clone() for layer in layers})) + reference[s] = entry + result["reference_s"] = round(time.time() - t0, 1) + finally: + for h in hooks: + h.remove() + result["rss_after_reference_gb"] = round(_rss_gb(), 1) + logger.info(f"batch decode: reference of {len(ref_slots)} slots in {result['reference_s']} s") + + # --- the device: build, prefill every slot into the pool --- + t0 = time.time() + engram, engram_hash = build_engram_modules(mesh_device, C, ckpt, layers, tokenizer) + top = ckpt.read(["embed.weight", "norm.weight", "head.weight"]) + model = TtV41Transformer( + mesh_device, + C, + layers, + lambda layer, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, layer), + top["embed.weight"], + top["norm.weight"], + top["head.weight"], + max_seq_len=max_seq, + chunk=chunk, + engram=engram, + engram_hash=engram_hash, + weight_cache_path=root, + ) + decoder = TtV41BatchDecoder(model, root, users_per_rank=users, trace=trace, warmup=warmup, sample_on_device=sample) + assert not (sample and reference), "device sampling returns ids: no logits to compare with the reference" + result["build_s"] = round(time.time() - t0, 1) + result["dram_after_build_bytes"] = _dram_bytes(mesh_device) + + t0 = time.time() + prefill_argmax = {} + for s in range(n_slots): + logits, state = model.prefill(prompts[s], logit_positions=1) + assert not torch.isnan(logits[-1]).any(), f"NaN in the prefill logits of slot {s}" + prefill_argmax[s] = int(logits[-1].argmax()) + if s in reference: + reference[s]["prefill_pcc"] = round(_pcc(logits[-1], reference[s]["prefill"]), 5) + decoder.join(s, prompts[s], state) + if s == 0 and os.environ.get("V41_BATCH_DEBUG", "0") == "1": + result["debug_join"] = _debug_join(decoder, state, layers) + logger.info(f"batch decode: join check {result['debug_join']}") + _release(state) + result["prefill_join_s"] = round(time.time() - t0, 1) + result["prefill_logits_pcc"] = {str(s): reference[s]["prefill_pcc"] for s in reference} + result["dram_after_join_bytes"] = _dram_bytes(mesh_device) + logger.info(f"batch decode: {n_slots} slots joined in {result['prefill_join_s']} s, {result['prefill_logits_pcc']}") + + # --- decode, all slots per step --- + captured = {} + if reference: + decoder.on_block = lambda layer, x: captured.__setitem__(layer, x) + if profile: + ttnn.ReadDeviceProfiler(mesh_device) + previous = decoder.on_block + + def drain(layer, x): + if previous is not None: + previous(layer, x) + ttnn.ReadDeviceProfiler(mesh_device) + + decoder.on_block = drain + per_step = [] + hits = {s: 0 for s in range(n_slots)} + assert not (live and reference), "V41_BATCH_LIVE steps the slots in turn: no per-step reference" + counter = {s: 0 for s in range(n_slots)} # tokens each slot has stepped (the live mode) + for i in range(steps): + if live and i >= live_from: + stepping = [(((i - live_from) * live if live_rotate else 0) + j) % n_slots for j in range(live)] + tokens = {s: int(books[s][counter[s]]) for s in stepping} + else: + stepping = list(range(n_slots)) + tokens = {s: int(books[s][i]) for s in range(n_slots)} + captured.clear() + t0 = time.time() + if sample and i == 0 and os.environ.get("V41_BATCH_DEBUG", "0") == "1": + decoder.sampler.debug = {} + if profile and i == steps - 1: + signpost("start") + out = decoder.decode(tokens, {s: (1, 0.0, 0.0) for s in tokens} if sample else None) + wall = time.time() - t0 + if profile: + ttnn.ReadDeviceProfiler(mesh_device) + if i == steps - 1: + signpost("stop") + if sample and decoder.sampler.debug: + dbg = decoder.sampler.debug + decoder.sampler.debug = None + for s in (0, 1, 4, 5, 9, 31): + r, u = divmod(s, users) + chips = [ + ttnn.to_torch(ttnn.get_device_tensors(dbg["logits"])[r * tp + j])[0, 0, u].float() + for j in range(tp) + ] + full = torch.cat(chips) + vals = ttnn.to_torch(ttnn.get_device_tensors(dbg["values"])[r * tp])[0, 0, u].float() + idx = ttnn.to_torch(ttnn.get_device_tensors(dbg["indices"])[r * tp])[0, 0, u].to(torch.int64) + ids_r = ttnn.to_torch(ttnn.get_device_tensors(dbg["ids"])[r * tp])[0, 0, 0].to(torch.int64) + logger.info( + f"sample debug slot {s} (rank {r} row {u}): host argmax {int(full.argmax())} ({full.max():.2f}); " + f"candidate max {vals.max():.2f} at {int(idx[vals.argmax()])}; sampled {int(ids_r[u])}; " + f"out {out[s]}; per-chip argmax {[int(c.argmax()) + j * (C.VOCAB_SIZE // tp) for j, c in enumerate(chips)]}; " + f"cand idx[0:4] {idx[:4].tolist()} idx[32:36] {idx[32:36].tolist()}" + ) + entry = {"step": i, "wall_s": round(wall, 3), **decoder.last, "slots": {}} + if live: + entry["stepping"] = stepping + for s in stepping: + if sample: + dev = int(out[s]) + else: + logits = out[s] + assert not torch.isnan(logits).any(), f"NaN in the logits of slot {s} at step {i}" + dev = int(logits.argmax()) + n = counter[s] + hits[s] += dev == int(books[s][n + 1]) + counter[s] += 1 + if s in reference: + ref_logits, ref_blocks = reference[s]["steps"][i] + r, u = divmod(s, users) + entry["slots"][str(s)] = { + "dev_argmax": dev, + "ref_argmax": int(ref_logits.argmax()), + "logits_pcc": round(_pcc(logits, ref_logits), 5), + "top5_hit": dev in ref_logits.topk(5).indices.tolist(), + "block_pcc": { + str(layer): round( + _pcc(_row_to_host(captured[layer], r, u, tp, C.HC_MULT), ref_blocks[layer]), 5 + ) + for layer in layers + }, + } + per_step.append(entry) + logger.info(f"batch step {i}: {json.dumps({k: v for k, v in entry.items() if k != 'slots'})} {entry['slots']}") + result["steps_detail"] = per_step + replays = [e for e in per_step if e["mode"] == "replay"] + result["replay_steps"] = len(replays) + if replays: + result["trace_segments"] = decoder.trace_segments + result["replay_wall_min_s"] = min(e["wall_s"] for e in replays) + result["replay_device_min_s"] = min(e["device_s"] for e in replays) + for key in ("inputs_s", "stage_s", "sync_s", "issue_s", "read_s"): + result["replay_" + key] = [e.get(key) for e in replays] + result["replay_device_only_s"] = decoder.replay_device_time() + decoder.release() + result["book_top1_per_slot"] = {str(s): round(hits[s] / counter[s], 4) for s in range(n_slots) if counter[s]} + result["book_top1"] = round(sum(hits.values()) / sum(counter.values()), 4) + result["text"] = { + str(s): tokenizer.decode(books[s][1 : counter[s] + 1].tolist()) for s in (0, n_slots - 1) if counter[s] + } + if reference: + agg = {} + for s in reference: + rows = [e["slots"][str(s)] for e in per_step] + agg[str(s)] = { + "top1_agreement": round(sum(r["dev_argmax"] == r["ref_argmax"] for r in rows) / steps, 4), + "top5_agreement": round(sum(r["top5_hit"] for r in rows) / steps, 4), + "logits_pcc_min": min(r["logits_pcc"] for r in rows), + "block_pcc_min": {str(layer): min(r["block_pcc"][str(layer)] for r in rows) for layer in layers}, + } + result["reference_agreement"] = agg + result["logits_pcc_min"] = min(a["logits_pcc_min"] for a in agg.values()) + result["top1_agreement"] = round(sum(a["top1_agreement"] for a in agg.values()) / len(agg), 4) + result["top5_agreement"] = round(sum(a["top5_agreement"] for a in agg.values()) / len(agg), 4) + result["block_pcc_min"] = { + str(layer): min(a["block_pcc_min"][str(layer)] for a in agg.values()) for layer in layers + } + result["dram_after_decode_bytes"] = _dram_bytes(mesh_device) + result["rss_peak_gb"] = round(_rss_gb(), 1) + logger.info("V41_BATCH_RESULT " + json.dumps(result)) + out_path = os.environ.get("V41_BATCH_OUT") + if out_path: + with open(out_path, "w") as f: + json.dump(result, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_block_4x2.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_block_4x2.py new file mode 100644 index 000000000000..678b8252d3c3 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_block_4x2.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""``test_block_v41`` on the LoudBox 4x2 mesh (SP4 x TP2; dev-spec §8: block tests on 2x4 and 4x2). + +Same cases, bars and checks as ``test_v41_blocks_on_device_state`` (called as is); only the mesh differs, which +exercises the TP=2 splits (heads, groups, hidden) and SP=4 cache geometry that 2x4 does not. + +The device MoE weight cache holds per-mesh-coordinate shards (a 2x4 cache fails to load on 4x2: coordinate [0, 2] +is not in a 4x2 mesh), so this test keeps its own cache root under ``test_block_v41``'s. +""" + +import pytest + +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41 import test_block_v41 as blocks + + +@pytest.mark.timeout(5400) +@pytest.mark.parametrize("kv_format", list(blocks.KV_FORMATS)) +@pytest.mark.parametrize("prompt", ["full", "padded", "tiny"]) +@pytest.mark.parametrize("schedule", ["stack", "swa"]) +@pytest.mark.parametrize("chunks", [1, 2], ids=["single_chunk", "two_chunks"]) +@pytest.mark.parametrize("weights", ["small", "synthetic", "real"]) +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (4, 2), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(4, 2), topology="mesh-4x2"), + id="fabric2d-mesh-4x2", + ) + ], + indirect=True, +) +def test_v41_blocks_on_device_state_4x2(mesh_device, device_params, weights, chunks, schedule, prompt, kv_format): + blocks.test_v41_blocks_on_device_state(mesh_device, device_params, weights, chunks, schedule, prompt, kv_format) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_block_overlap_experiments.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_block_overlap_experiments.py new file mode 100644 index 000000000000..499b9fac97ee --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_block_overlap_experiments.py @@ -0,0 +1,490 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Block op-overlap experiments E0, E1, E3 (bead 8y7.9.11; design study ``block-overlap-design.md`` §6). + +LoudBox 2x4 (SP = 2, TP = 4), traced replay, device-synchronized wall time (the traced-span domain of the G2 profile). +Each case logs one ``OVERLAP {json}`` line; the assertions only guard that the experiment ran and measured what it +names (the numbers are results, not contracts): + +* **E0** (sub-device switch cost): the same matmuls on an 11 x 8 grid, as (a) one trace, (b) split into trace segments + with no manager switch, (c) segments with a load / clear of a 2 / 8-row manager (the Plan B partition) around each + second matmul, replayed like ``SubDeviceTraceController`` (non-blocking segments, host load / clear between them). + Cost per switch pair = (c - a) / pairs; per segment boundary = (b - a) / boundaries. +* **E1** (issue order): two 55-core sub-devices (rows 0-4 / 5-9), chains of 3 dependent compute-bound matmuls, issued + grouped (A1 A2 A3 B1 B2 B3) vs interleaved (A1 B1 A2 B2 A3 B3), plus one long op then a chain (the TtMoe pattern). + Concurrent chains take about max(A, B); lock-step issue (hypothesis H-CQ) makes grouped take about A + 2/3 B. +* **E3** (fused wo_b + TP reduce-scatter): per chip [2560, 2048] x [2048, 5120] (bf16 x bfp8, HiFi2) then the TP + reduce-scatter of dim 3, as today (``ttnn.linear`` + ``V41Collectives.tp_reduce_scatter``, Linear) vs + ``minimal_matmul_strided_reduce_scatter_async`` (Ring only; run only on a ring TP axis, see the test). Accuracy vs + the fp32 product of the device inputs, determinism by repeat, time. +* **E3b** (Plan A2 without a fused collective): wq_a and wkv as one matmul and one TP all-reduce plus two slices, vs + two of each. Bit-equality of the matmul columns, accuracy, time. +""" + +import json +import time + +import pytest +import torch +from loguru import logger +from tracy import signpost + +import ttnn +from models.common.timing_events import phase +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_trace import MESH +from models.demos.deepseek_v3_d_p.tt.v41.attention import DENSE_COMPUTE_CONFIG +from models.demos.deepseek_v3_d_p.tt.v41.ccl import V41Collectives +from models.demos.deepseek_v3_d_p.tt.v41.layout import TP_AXIS +from tests.ttnn.utils_for_testing import comp_pcc + +REPLAYS = 10 +ROUNDS = 3 + + +def _log(**rec): + logger.info(f"OVERLAP {json.dumps(rec)}") + + +def _tensor(mesh_device, shape, dtype=ttnn.bfloat16, scale=1.0, mapper=None): + return ttnn.from_torch( + (torch.randn(shape) * scale).to(torch.bfloat16), + device=mesh_device, + dtype=dtype, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=mapper or ttnn.ReplicateTensorToMesh(mesh_device), + ) + + +def _rows(x0, y0, x1, y1): + return ttnn.CoreRangeSet({ttnn.CoreRange(ttnn.CoreCoord(x0, y0), ttnn.CoreCoord(x1, y1))}) + + +class _Segmented: + """Capture of a step list into trace segments split at ``("load", id)`` / ``("clear",)`` / ``("split",)`` + markers; replay as ``SubDeviceTraceController.replay`` does (segments non-blocking, host load / clear between).""" + + def __init__(self, mesh_device, steps): + self.mesh_device, self.program, self.keep = mesh_device, [], [] + tid = None + for step in steps: + if callable(step): + if tid is None: + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + self.keep.append(step()) + continue + if tid is not None: + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + self.program.append(("trace", tid)) + tid = None + if step[0] == "load": + mesh_device.load_sub_device_manager(step[1]) + elif step[0] == "clear": + mesh_device.clear_loaded_sub_device_manager() + if step[0] != "split": + self.program.append(step) + if tid is not None: + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + self.program.append(("trace", tid)) + + def replay(self): + for step in self.program: + if step[0] == "trace": + ttnn.execute_trace(self.mesh_device, step[1], cq_id=0, blocking=False) + elif step[0] == "load": + self.mesh_device.load_sub_device_manager(step[1]) + else: + self.mesh_device.clear_loaded_sub_device_manager() + + def timed_us(self, replays=REPLAYS): + """Mean wall per replay of ``replays`` back-to-back replays (one device synchronize), after a warm one.""" + self.replay() + ttnn.synchronize_device(self.mesh_device) + t0 = time.perf_counter() + for _ in range(replays): + self.replay() + ttnn.synchronize_device(self.mesh_device) + return (time.perf_counter() - t0) * 1e6 / replays + + def release(self): + for step in self.program: + if step[0] == "trace": + ttnn.release_trace(self.mesh_device, step[1]) + self.keep.clear() + + +def _mm(a, b, grid, sd=None): + kwargs = {} if sd is None else {"sub_device_id": sd} + return ttnn.matmul( + a, + b, + core_grid=grid, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + compute_kernel_config=DENSE_COMPUTE_CONFIG, + **kwargs, + ) + + +# --- E0 ------------------------------------------------------------------------------------------------------------ +E0_PAIRS = 8 +E0_M = 4096 # rows of each E0 matmul: about 0.4 ms on 88 cores + + +@pytest.mark.timeout(900) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_e0_sub_device_switch_cost(mesh_device, device_params): + grid = mesh_device.compute_with_storage_grid_size() + gx, gy = grid.x, grid.y + small_rows = 2 + mgr = mesh_device.create_sub_device_manager( + [ttnn.SubDevice([_rows(0, 0, gx - 1, small_rows - 1)]), ttnn.SubDevice([_rows(0, small_rows, gx - 1, gy - 1)])], + 0, + ) + big = ttnn.SubDeviceId(1) + core_grid = ttnn.CoreGrid(x=gx, y=gy - small_rows) + with phase("weights", test="e0"): + x = _tensor(mesh_device, (1, 1, E0_M, 2048)) + w = _tensor(mesh_device, (1, 1, 2048, 2048), dtype=ttnn.bfloat8_b, scale=2048**-0.5) + full = lambda: _mm(x, w, core_grid) + on_sd = lambda: _mm(x, w, core_grid, big) + results = {} + with phase("compute", test="e0"): + # compile both program variants (full-grid manager, and the split manager) + full() + mesh_device.load_sub_device_manager(mgr) + on_sd() + mesh_device.clear_loaded_sub_device_manager() + ttnn.synchronize_device(mesh_device) + variants = { + "one_trace": [full] * (2 * E0_PAIRS), + "split_only": sum([[full, full, ("split",)] for _ in range(E0_PAIRS)], [])[:-1], + "switch": sum([[full, ("load", mgr), on_sd, ("clear",)] for _ in range(E0_PAIRS)], []), + } + segs = {name: _Segmented(mesh_device, steps) for name, steps in variants.items()} + for _ in range(ROUNDS): # rounds interleave the variants, so drift hits all of them + for name, seg in segs.items(): + results[name] = min(results.get(name, float("inf")), seg.timed_us()) + for name, seg in segs.items(): + results[f"{name}_segments"] = sum(s[0] == "trace" for s in seg.program) + seg.release() + mesh_device.remove_sub_device_manager(mgr) + per_mm = results["one_trace"] / (2 * E0_PAIRS) + switch_pair = (results["switch"] - results["one_trace"]) / E0_PAIRS + boundary = (results["split_only"] - results["one_trace"]) / (E0_PAIRS - 1) + # the switch variant has 2 * pairs segments: 2 * pairs - 1 boundaries, each a host load or clear + per_switch_boundary = (results["switch"] - results["one_trace"]) / (2 * E0_PAIRS - 1) + _log( + exp="E0", + pairs=E0_PAIRS, + grid=[gx, gy], + partition=f"rows 0-{small_rows - 1} / {small_rows}-{gy - 1}", + per_matmul_us=per_mm, + switch_pair_us=switch_pair, + segment_boundary_us=boundary, + per_switch_boundary_us=per_switch_boundary, + **results, + ) + assert results["switch_segments"] == 2 * E0_PAIRS + + +# --- E1 ------------------------------------------------------------------------------------------------------------ +E1_M, E1_K = 2560, 3072 + + +@pytest.mark.timeout(900) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_e1_issue_order(mesh_device, device_params): + grid = mesh_device.compute_with_storage_grid_size() + gx, gy = grid.x, grid.y + half = gy // 2 + mgr = mesh_device.create_sub_device_manager( + [ttnn.SubDevice([_rows(0, 0, gx - 1, half - 1)]), ttnn.SubDevice([_rows(0, half, gx - 1, gy - 1)])], 0 + ) + sd = [ttnn.SubDeviceId(0), ttnn.SubDeviceId(1)] + core_grid = ttnn.CoreGrid(x=gx, y=half) + with phase("weights", test="e1"): + xs = [_tensor(mesh_device, (1, 1, E1_M, E1_K)) for _ in sd] + x_long = _tensor(mesh_device, (1, 1, 3 * E1_M, E1_K)) + w = _tensor(mesh_device, (1, 1, E1_K, E1_K), dtype=ttnn.bfloat8_b, scale=E1_K**-0.5) + + def chain(side): + """The 3 steps of chain ``side``; each consumes the previous output (a real dependency).""" + state = {"t": xs[side]} + + def step(): + state["t"] = _mm(state["t"], w, core_grid, sd[side]) + return state["t"] + + return [step] * 3 + + long_op = lambda: _mm(x_long, w, core_grid, sd[0]) + orders = { + "A_alone": lambda: chain(0), + "B_alone": lambda: chain(1), + "grouped": lambda: chain(0) + chain(1), + "interleaved": lambda: [s for pair in zip(chain(0), chain(1)) for s in pair], + "long_alone": lambda: [long_op], + "long_then_chain": lambda: [long_op] + chain(1), + "chain_then_long": lambda: chain(1) + [long_op], + } + results = {} + with phase("compute", test="e1"): + # every replay is load, body, clear: the clear waits for both sub-devices, so replays do not overlap each + # other and each variant carries the same switch cost (E0) + segs = {} + for name, make in orders.items(): + mesh_device.load_sub_device_manager(mgr) + for step in make(): # compile / warm + step() + mesh_device.clear_loaded_sub_device_manager() + ttnn.synchronize_device(mesh_device) + segs[name] = _Segmented(mesh_device, [("load", mgr)] + make() + [("clear",)]) + segs["switch_only"] = _Segmented(mesh_device, [("load", mgr), ("clear",)]) + for _ in range(ROUNDS): + for name, seg in segs.items(): + results[name] = min(results.get(name, float("inf")), seg.timed_us()) + for seg in segs.values(): + seg.release() + mesh_device.remove_sub_device_manager(mgr) + a, b = results["A_alone"], results["B_alone"] + _log( + exp="E1", + m=E1_M, + k=E1_K, + cores_per_sd=gx * half, + serial_sum_us=a + b, + concurrent_bound_us=max(a, b), + lockstep_prediction_us=a + 2 * b / 3, + **results, + ) + assert all(v > 0 for v in results.values()) + + +# --- E3 ------------------------------------------------------------------------------------------------------------ +E3_M, E3_K, E3_N = 2560, 2048, 5120 # per chip: rows = chunk 5120 / SP, K = O_GROUPS * O_LORA_RANK / TP, N = hidden +E3_ITERS = 8 +# (grid x, grid y, M block, K block, N block, subblock h, subblock w, chunk width in mm blocks), in tiles +E3_CONFIGS = [ + (10, 8, 5, 8, 8, 1, 8, 1), + (10, 8, 10, 4, 8, 2, 4, 1), + (10, 8, 5, 8, 16, 1, 8, 1), + (10, 8, 5, 8, 8, 1, 8, 2), + (8, 8, 5, 8, 10, 1, 5, 1), +] + + +def _timed_trace_us(mesh_device, fn, iters=E3_ITERS, label=None): + """Min over replays of the per-call wall; under the device profiler, ``label`` signposts the replays (the ops + CSV then attributes kernel durations per case) and the profiler buffer is read after them.""" + fn() + ttnn.synchronize_device(mesh_device) + if label: + signpost(label) + keep = [] + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + for _ in range(iters): + keep.append(fn()) + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=True) + best = float("inf") + for _ in range(REPLAYS): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=True) + best = min(best, (time.perf_counter() - t0) * 1e6 / iters) + ttnn.release_trace(mesh_device, tid) + if label: + ttnn.ReadDeviceProfiler(mesh_device) + return best + + +@pytest.mark.timeout(1200) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_e3_wo_b_fused_reduce_scatter(mesh_device, device_params): + assert tuple(mesh_device.shape) == (2, 4) + tp = mesh_device.shape[TP_AXIS] + coll = V41Collectives(mesh_device) + shard = lambda dim: ttnn.ShardTensor2dMesh(mesh_device, tuple(mesh_device.shape), dims=(None, dim)) + torch.manual_seed(0) + with phase("weights", test="e3"): + a = _tensor(mesh_device, (1, 1, E3_M, E3_K * tp), mapper=shard(3)) + w = _tensor( + mesh_device, (1, 1, E3_K * tp, E3_N), dtype=ttnn.bfloat8_b, scale=(E3_K * tp) ** -0.5, mapper=shard(2) + ) + with phase("oracle", test="e3"): + dev = lambda t: [ttnn.to_torch(d).float() for d in ttnn.get_device_tensors(t)] + a_c, w_c = dev(a)[:tp], dev(w)[:tp] # mesh row 0; row 1 holds the same shards + golden = sum(x @ y for x, y in zip(a_c, w_c))[0, 0] + slice_n = E3_N // tp + want = [golden[:, c * slice_n : (c + 1) * slice_n] for c in range(tp)] + + def check(name, out): + got = dev(out) + assert len(got) == 2 * tp and list(got[0].shape) == [1, 1, E3_M, slice_n], (name, got[0].shape) + pccs = [comp_pcc(want[i % tp], got[i][0, 0], 0.0)[1] for i in range(len(got))] + return min(pccs), got + + results = {} + with phase("compute", test="e3"): + base = lambda: coll.tp_reduce_scatter(ttnn.linear(a, w, compute_kernel_config=DENSE_COMPUTE_CONFIG)) + mm_only = lambda: ttnn.linear(a, w, compute_kernel_config=DENSE_COMPUTE_CONFIG) + pcc, ref = check("baseline", base()) + repeat = torch.equal(torch.stack([g for g in dev(base())]), torch.stack(ref)) + results["baseline"] = dict(us=_timed_trace_us(mesh_device, base), pcc=pcc, deterministic=repeat) + results["baseline_mm"] = dict(us=_timed_trace_us(mesh_device, mm_only)) + rs_in = mm_only() + results["baseline_rs"] = dict(us=_timed_trace_us(mesh_device, lambda: coll.tp_reduce_scatter(rs_in))) + _log(exp="E3", case="baseline", **results["baseline"], mm_us=results["baseline_mm"]["us"]) + + # The fused op validates Ring only. On the LoudBox (FABRIC_2D, TP axis not wrapped) its first call hung the + # device (dispatch timeout, 2026-09-30, bead 8y7.9.11), so it runs only where the TP axis is a ring. + configs = E3_CONFIGS if coll.tp_topology == ttnn.Topology.Ring else [] + if not configs: + _log(exp="E3", case="fused", skipped=f"TP topology {coll.tp_topology} is not Ring") + for gx, gy, mb, kb, nb, sh, sw, cw in configs: + name = f"fused_g{gx}x{gy}_m{mb}k{kb}n{nb}_s{sh}x{sw}_cw{cw}" + cfg = ttnn.MinimalMatmulConfig( + M_block_size=mb, + K_block_size=kb, + N_block_size=nb, + subblock_h=sh, + subblock_w=sw, + compute_with_storage_grid_size=ttnn.CoreCoord(gx, gy), + ) + + def fused(cfg=cfg, gy=gy, cw=cw): + _, rs = ttnn.experimental.minimal_matmul_strided_reduce_scatter_async( + a, + w, + 3, + coll.tt_ccl.get_and_cycle_rs_semaphore_handles(cluster_axis=TP_AXIS), + ttnn.CoreCoord(0, gy), + compute_kernel_config=DENSE_COMPUTE_CONFIG, + num_links=coll.num_links, + memory_config_mm=ttnn.DRAM_MEMORY_CONFIG, + rs_output_mem_config=ttnn.DRAM_MEMORY_CONFIG, + topology=ttnn.Topology.Ring, + cluster_axis=TP_AXIS, + config=cfg, + barrier_semaphore=coll.tt_ccl.get_and_cycle_barrier_semaphore_handle(cluster_axis=TP_AXIS), + chunk_width_in_mm_blocks=cw, + ) + return rs + + try: + pcc, first = check(name, fused()) + except Exception as err: # config rejected by the op's validation: record and go on + results[name] = dict(error=str(err).splitlines()[0][:300]) + _log(exp="E3", case=name, **results[name]) + continue + runs = [torch.stack(dev(fused())) for _ in range(3)] + deterministic = all(torch.equal(r, torch.stack(first)) for r in runs) + vs_base = max((g - r).abs().max().item() for g, r in zip(first, ref)) + results[name] = dict( + us=_timed_trace_us(mesh_device, fused), pcc=pcc, deterministic=deterministic, max_abs_vs_base=vs_base + ) + _log(exp="E3", case=name, **results[name]) + _log(exp="E3", case="summary", results=results) + assert results["baseline"]["deterministic"] + + +# --- E3b: Plan A2 without a fused collective (merged wq_a + wkv, one TP all-reduce) -------------------------------- +A2_M, A2_K, A2_NQ, A2_NKV = 2560, 1280, 1280, 512 # per chip: rows, hidden / TP, q_lora_rank, head_dim + + +def _merged_grid_sweep(mesh_device, x, w, block, reference): + """Time the merged [M, K] x [K, NQ + NKV] matmul on narrower 2D-multicast grids (same K block, so the same + per-tile accumulation order); returns {grid: us} and whether each is bit-equal to the default-grid result.""" + mt, nt = A2_M // 32, w.shape[-1] // 32 + out = {} + for gx, gy in ((7, 10), (8, 10), (11, 8), (7, 8)): + pm, pn = -(-mt // gy), -(-nt // gx) + sub_w = max(v for v in range(1, 9) if pn % v == 0) + sub_h = max(h for h in range(1, 9) if pm % h == 0 and h * sub_w <= 8) + cfg = ttnn.MatmulMultiCoreReuseMultiCastProgramConfig( + compute_with_storage_grid_size=(gx, gy), + in0_block_w=block, + out_subblock_h=sub_h, + out_subblock_w=sub_w, + per_core_M=pm, + per_core_N=pn, + transpose_mcast=False, + fuse_batch=False, + fused_activation=None, + ) + run = lambda cfg=cfg: ttnn.linear(x, w, program_config=cfg, compute_kernel_config=DENSE_COMPUTE_CONFIG) + got = [ttnn.to_torch(d).float() for d in ttnn.get_device_tensors(run())] + out[f"merged_mm_{gx}x{gy}_us"] = _timed_trace_us(mesh_device, run, label=f"merged_mm_{gx}x{gy}") + out[f"merged_mm_{gx}x{gy}_bit_equal"] = all(torch.equal(a, b) for a, b in zip(got, reference)) + return out + + +@pytest.mark.timeout(900) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_e3b_merged_q_kv_projection(mesh_device, device_params): + """wq_a and wkv share their input and are both row-parallel: one [K, NQ + NKV] matmul and one all-reduce, then + two slices, vs today's two matmuls and two all-reduces (``tt/v41/attention.py`` forward).""" + from models.demos.deepseek_v3_d_p.tt.v41.attention import DENSE_IN0_BLOCK_W, dense_program_config + + tp = mesh_device.shape[TP_AXIS] + coll = V41Collectives(mesh_device) + shard = lambda dim: ttnn.ShardTensor2dMesh(mesh_device, tuple(mesh_device.shape), dims=(None, dim)) + torch.manual_seed(0) + with phase("weights", test="e3b"): + x = _tensor(mesh_device, (1, 1, A2_M, A2_K * tp), mapper=shard(3)) + wq_t = torch.randn(1, 1, A2_K * tp, A2_NQ) * (A2_K * tp) ** -0.5 + wkv_t = torch.randn(1, 1, A2_K * tp, A2_NKV) * (A2_K * tp) ** -0.5 + mk = lambda t: ttnn.from_torch( + t.to(torch.bfloat16), + device=mesh_device, + dtype=ttnn.bfloat8_b, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=shard(2), + ) + wq, wkv, wqkv = mk(wq_t), mk(wkv_t), mk(torch.cat([wq_t, wkv_t], dim=-1)) + assert DENSE_IN0_BLOCK_W["wq_a"] == DENSE_IN0_BLOCK_W["wkv"] + block = DENSE_IN0_BLOCK_W["wq_a"] + dense = lambda w: ttnn.linear( + x, + w, + program_config=dense_program_config(A2_M, A2_K, w.shape[-1], block), + compute_kernel_config=DENSE_COMPUTE_CONFIG, + ) + separate = lambda: (coll.tp_all_reduce(dense(wq)), coll.tp_all_reduce(dense(wkv))) + + def merged(): + both = coll.tp_all_reduce(dense(wqkv)) + return ( + ttnn.slice(both, [0, 0, 0, 0], [1, 1, A2_M, A2_NQ]), + ttnn.slice(both, [0, 0, 0, A2_NQ], [1, 1, A2_M, A2_NQ + A2_NKV]), + ) + + dev = lambda t: [ttnn.to_torch(d).float() for d in ttnn.get_device_tensors(t)] + with phase("oracle", test="e3b"): + xs = dev(x)[:tp] + wq_d, wkv_d = dev(wq)[:tp], dev(wkv)[:tp] + want = [sum(a @ b for a, b in zip(xs, ws)) for ws in (wq_d, wkv_d)] + with phase("compute", test="e3b"): + sep, mer = [[dev(t) for t in f()] for f in (separate, merged)] + pcc = lambda outs: min(comp_pcc(want[i], o, 0.0)[1] for i in range(2) for o in outs[i]) + bit_equal = [all(torch.equal(a, b) for a, b in zip(sep[i], mer[i])) for i in range(2)] + max_abs = [max((a - b).abs().max().item() for a, b in zip(sep[i], mer[i])) for i in range(2)] + mm_equal = [ + all(torch.equal(a, b[..., sl]) for a, b in zip(dev(dense(w)), dev(dense(wqkv)))) + for w, sl in ((wq, slice(0, A2_NQ)), (wkv, slice(A2_NQ, None))) + ] + rec = dict( + separate_us=_timed_trace_us(mesh_device, separate, label="separate"), + merged_us=_timed_trace_us(mesh_device, merged, label="merged"), + separate_mm_us=_timed_trace_us(mesh_device, lambda: (dense(wq), dense(wkv)), label="separate_mm"), + merged_mm_us=_timed_trace_us(mesh_device, lambda: dense(wqkv), label="merged_mm"), + **_merged_grid_sweep(mesh_device, x, wqkv, block, dev(dense(wqkv))), + pcc_separate=pcc(sep), + pcc_merged=pcc(mer), + matmul_bit_equal_q_kv=mm_equal, + all_reduce_bit_equal_q_kv=bit_equal, + max_abs_q_kv=max_abs, + ) + _log(exp="E3b", **rec) + assert rec["pcc_merged"] > 0.999 diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_c1_index.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_c1_index.py new file mode 100644 index 000000000000..d441a61ec3af --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_c1_index.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The c1 capture's index selection for the live user alone (``V41_DECODE_C1_INDEX=1``): +the pieces that let the capture carry no slot, each bit for bit against the per-user path they replace. + +No weights; four users of a 20,480-row pool with FP4-like keys and queries as ``test_indexer.py``'s range +test: + +- ``range_scores(dyn=True)`` (the user and its newest read from the ``c1usr`` page) against ``range_scores`` +with + the user as an argument, for a live user at every slot, an idle slot and the no-live-row page; +- ``candidate_scores(base_page=...)`` (the slot's block page read from the page) against ``block_base``; +- the one-hot row pick (``C1Moe.onehots`` of the ``c1all`` row, a HiFi4 matmul) against the query's rows; +- ``c1_place`` (the selection into the live row, sentinels elsewhere) against torch, for every row and for +none; +- ``TtV41Indexer.select_split`` on the dyn score against the per-user score: the top-k and the published +blocks. + +Then the dyn scorer under trace (``V41_CI_REPLAYS`` replays) for its device time against the four per-user +launches. Results: ``V41_CI_OUT``.""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.test_indexer import _fp4_like, _replicated +from models.demos.deepseek_v3_d_p.tt.v41.batch_decode import _C1_HIFI4, c1_place +from models.demos.deepseek_v3_d_p.tt.v41.indexer import SENTINEL, TtV41Indexer +from models.demos.deepseek_v3_d_p.tt.v41.indexer_kernels import candidate_scores, range_scores +from models.demos.deepseek_v3_d_p.tt.v41.moe_c1 import _SENTINEL as ONEHOT_SENTINEL +from models.demos.deepseek_v3_d_p.tt.v41.moe_c1 import C1Moe + +ROWS = 32 + + +def _all_chips_equal(t, what: str) -> torch.Tensor: + devs = [ttnn.to_torch(d) for d in ttnn.get_device_tensors(t)] + for i, other in enumerate(devs[1:], 1): + assert torch.equal(other, devs[0]), f"{what}: chip {i} differs from chip 0" + return devs[0][0, 0] + + +def _bits(t: torch.Tensor) -> torch.Tensor: + return t.view(torch.int16) if t.dtype == torch.bfloat16 else t + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=64 * 1024 * 1024), indirect=True) +def test_v41_c1_index(mesh_device, device_params): + replays = int(os.environ.get("V41_CI_REPLAYS", 20)) + cfg = C + cap = 20480 + width = cap + 32 + users = 4 + nb = width // cfg.CANDIDATE_BLOCK_SIZE # block pages a user + g = torch.Generator().manual_seed(86) + newest = torch.full((ROWS,), -1, dtype=torch.int64) + newest[:users] = torch.tensor([19999, 299, -1, cap - 1]) # a long prefix, a short one, idle, the full pool + q = _fp4_like((ROWS, 32, 128), g) + w = (torch.randn(ROWS, 32, generator=g) * 0.05).to(torch.bfloat16) + k = _fp4_like((users * width, 128), g) + k.view(users, width, 128)[:, cap:] = 0 + dev_q = _replicated(mesh_device, q.reshape(1, 1, ROWS, -1), ttnn.bfloat16) + dev_w = _replicated(mesh_device, w.reshape(1, 1, ROWS, 32), ttnn.bfloat16) + dev_k = ttnn.from_torch( + k.reshape(users, 1, width, 128), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + dev_pin = _replicated(mesh_device, newest.view(1, 1, 1, ROWS).to(torch.int32), ttnn.uint32) + blocks = _replicated(mesh_device, k.reshape(1, 1, users * nb, cfg.CANDIDATE_BLOCK_SIZE * 128), ttnn.bfloat16) + table = C1Moe.identity_table(mesh_device) + result = {"cap": cap, "users": users, "cases": []} + + def usr_page(live: int | None): + """The ``c1usr`` row of ``_host_inputs``: [user, block page of its block 0, newest, sentinel...].""" + page = torch.full((1, 1, 1, ROWS), -1, dtype=torch.int32) + page[..., :2] = 0 + if live is not None: + page[..., 0], page[..., 1], page[..., 2] = live, live * nb, int(newest[live]) + return _replicated(mesh_device, page, ttnn.uint32) + + def pin_page(live: int | None): + """The ``c1pin`` row: the live user's newest in every word.""" + page = torch.full((1, 1, 1, ROWS), -1 if live is None else int(newest[live]), dtype=torch.int32) + return _replicated(mesh_device, page, ttnn.uint32) + + # 1. the dyn scorer against the per-user scorer, bit for bit, every chip + for live in [0, 1, 2, 3, None]: + usr = usr_page(live) + dyn = _all_chips_equal(range_scores(dev_q, dev_w, dev_k, usr, 0, width, dyn=True), f"dyn live {live}") + ref = _all_chips_equal(range_scores(dev_q, dev_w, dev_k, dev_pin, live or 0, width), f"static {live}") + if live is None: # no live row: every score -inf, as an idle user's + assert bool((dyn == float("-inf")).all()), "the no-live-row page scored something" + else: + assert torch.equal(_bits(dyn), _bits(ref)), f"range_scores dyn differs from static for user {live}" + finite = int(torch.isfinite(dyn).sum()) + logger.info(f"range_scores dyn: live {live}: {finite} finite scores, bit-equal to the per-user launch") + result["cases"].append({"range_scores": live, "finite": finite}) + + # 2. the candidate scorer with the base page against block_base, on random blocks of each user (sentinel + # tail) + source = TtV41Indexer( + mesh_device, + cfg, + cfg.CANDIDATE_SOURCE_LAYER, + { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + }, + ) + consumer = TtV41Indexer( + mesh_device, + cfg, + 24, + { + "wq_b": torch.zeros(cfg.INDEX_N_HEADS * cfg.INDEX_HEAD_DIM, cfg.Q_LORA_RANK), + "weights_proj": torch.zeros(cfg.INDEX_N_HEADS, cfg.EMB_SIZE), + }, + ) + kb = cfg.CANDIDATE_TOPK_BLOCKS + topk = cfg.INDEX_TOPK + for live in [0, 1, 3]: + n_u = int(newest[live]) + ids = torch.randperm(max(1, (n_u + 1) // cfg.CANDIDATE_BLOCK_SIZE), generator=g)[:kb] + ids = ( + torch.cat([ids, torch.full((kb - ids.numel(),), SENTINEL, dtype=torch.int64)]) if ids.numel() < kb else ids + ) + ids_u = _replicated(mesh_device, ids.view(1, 1, 1, kb).to(torch.int32), ttnn.uint32) + # the one-hot rows (c1all: the live user in every word) pick the query and weights into every row + all_ = _replicated(mesh_device, torch.full((1, 1, 1, ROWS), live, dtype=torch.int32), ttnn.uint32) + sel_all, _ = C1Moe.onehots(all_, all_, table) + q_t = ttnn.to_layout(dev_q, ttnn.TILE_LAYOUT) + w_t = ttnn.to_layout(dev_w, ttnn.TILE_LAYOUT) + q_n = ttnn.to_layout(ttnn.matmul(sel_all, q_t, compute_kernel_config=_C1_HIFI4), ttnn.ROW_MAJOR_LAYOUT) + w_n = ttnn.to_layout(ttnn.matmul(sel_all, w_t, compute_kernel_config=_C1_HIFI4), ttnn.ROW_MAJOR_LAYOUT) + q_pick = _all_chips_equal(q_n, "q one-hot") + w_pick = _all_chips_equal(w_n, "w one-hot") + # as values: the one-hot matmul turns a -0.0 of the FP4-like query into +0.0 (1.0 x -0.0 + 0.0), which + # no + # score can tell apart (relu(q . k) of either is the same bits) + assert torch.equal(q_pick.float(), q[live].reshape(1, -1).expand(ROWS, -1).float()), "q one-hot pick" + assert torch.equal(w_pick.float(), w[live].reshape(1, -1).expand(ROWS, -1).float()), "w one-hot pick" + pin_n = pin_page(live) + usr = usr_page(live) + dyn = consumer.select_live_in_blocks(q_n, w_n, blocks, pin_n, ids_u, usr, width, topk) + ref = consumer.select_user_in_blocks(dev_q, dev_w, blocks, dev_pin, ids_u, live, width, live * nb, topk) + dyn_t, ref_t = _all_chips_equal(dyn, "consumer dyn"), _all_chips_equal(ref, "consumer static") + assert torch.equal(dyn_t, ref_t), f"consumer selection differs for user {live}" + # and the scorer itself, bit for bit + n = ROWS + per = kb // n + ids_n = ttnn.reshape(ids_u, [1, 1, n, per]) + s_dyn = candidate_scores(q_n, w_n, blocks, ids_n, pin_n, 0, width, block_base=0, base_page=usr) + s_ref = candidate_scores(q_n, w_n, blocks, ids_n, pin_n, 0, width, block_base=live * nb) + assert torch.equal(_bits(_all_chips_equal(s_dyn, "cs dyn")), _bits(_all_chips_equal(s_ref, "cs static"))) + picked = int((dyn_t != SENTINEL).sum()) + logger.info(f"consumer: live {live}: {picked} rows selected, the dyn path bit-equal to the per-user path") + result["cases"].append({"consumer": live, "selected": picked}) + + # 3. the candidate source's split selection on the dyn score: top-k and published blocks as the + # per-user's + pin_u = ttnn.slice(usr, [0, 0, 0, 2], [1, 1, 1, 3]) + pin_s = ttnn.slice(dev_pin, [0, 0, 0, live], [1, 1, 1, live + 1]) + sc_dyn = range_scores(dev_q, dev_w, dev_k, usr, 0, width, dyn=True) + sc_ref = range_scores(dev_q, dev_w, dev_k, dev_pin, live, width) + t_dyn, p_dyn = source.select_split(sc_dyn, pin_u, width, topk) + t_ref, p_ref = source.select_split(sc_ref, pin_s, width, topk) + assert torch.equal(_all_chips_equal(t_dyn, "source dyn"), _all_chips_equal(t_ref, "source static")) + assert (p_dyn.ids is None) == (p_ref.ids is None) + if p_dyn.ids is not None: + assert torch.equal(_all_chips_equal(p_dyn.ids, "blocks dyn"), _all_chips_equal(p_ref.ids, "blocks static")) + logger.info(f"source: live {live}: the split selection and the published blocks bit-equal") + + # 4. the placement: a random selection with a sentinel tail into every row, and into none + sel = torch.randint(0, 1 << 20, (topk,), generator=g) + sel[topk - 7 :] = SENTINEL + sel_dev = _replicated(mesh_device, sel.view(1, 1, 1, topk).to(torch.int32), ttnn.uint32) + for u in list(range(ROWS)) + [None]: + put_row = torch.full((1, 1, 1, ROWS), ONEHOT_SENTINEL, dtype=torch.int32) + if u is not None: + put_row[..., u] = 0 + placed = _all_chips_equal(c1_place(sel_dev, _replicated(mesh_device, put_row, ttnn.uint32)), f"place {u}") + expect = torch.full((ROWS, topk), SENTINEL, dtype=torch.int64) + if u is not None: + expect[u] = sel + got = placed.to(torch.int64) & 0xFFFFFFFF + assert torch.equal(got, expect), f"placement into row {u}: {(got != expect).sum()} words differ" + logger.info("c1_place: every row and none, bit-exact (ids below 2^20, the sentinel tail kept)") + + # 5. device time under trace: the dyn scorer once against the four per-user launches + def timed(fn, n): + fn() + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + for _ in range(4): + fn() + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + t0 = time.time() + for _ in range(n): + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + per = (time.time() - t0) / (n * 4) + ttnn.release_trace(mesh_device, tid) + return per + + usr0 = usr_page(0) + t_dyn = timed(lambda: range_scores(dev_q, dev_w, dev_k, usr0, 0, width, dyn=True), replays) + t_static = timed(lambda: [range_scores(dev_q, dev_w, dev_k, dev_pin, u, width) for u in range(users)], replays) + result["trace_us"] = {"dyn_one_user": t_dyn * 1e6, "static_four_users": t_static * 1e6} + logger.info( + f"under trace: dyn scorer {t_dyn * 1e6:.1f} us a call, the four per-user launches {t_static * 1e6:.1f} us" + ) + out = os.environ.get("V41_CI_OUT") + if out: + with open(out, "w") as f: + json.dump(result, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_ccl_calibration.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_ccl_calibration.py new file mode 100644 index 000000000000..9279d8a4dab3 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_ccl_calibration.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""CCL link-rate calibration for the V4.1 perf model on the LoudBox 2x4 (bead 8y7.9.3). + +``utils/v41_perf_model.collective_ns`` charges ``latency + bottleneck-edge bytes / (rate x links)`` per collective +kind; its ``BLACKHOLE_CCL`` rates and latencies are fitted from this test (the uncalibrated 25 GB/s per link and +per-hop latency put the TP all-to-alls at 1.65-2.5x their "ideal"). This test times the collectives V4.1 uses, on +both mesh axes, at several per-chip sizes and 1 / 2 links, so ``rate`` and ``latency`` can be fitted per (kind, +axis, links) against the model's edge bytes. + +Collectives (bf16, DRAM, TILE, ``[1, 1, rows, 1024]`` per chip): + +* TP (axis 1, 4 chips): ``V41Collectives.tp_all_gather`` / ``tp_reduce_scatter`` / ``tp_all_to_all`` (dim 3 -> 2); +* SP (axis 0, 2 chips): ``all_gather_async`` and ``all_to_all_async_generic`` with ``cluster_axis=0``. + +Timing: ``ITERS`` back-to-back calls captured in one trace, one warm replay, then ``REPLAYS`` timed replays with a +device synchronize around them; time per call = wall / (REPLAYS x ITERS). Host dispatch is excluded by the trace; +inter-op gaps inside the trace are included (they are part of a traced layer too). One ``CCL_CAL {json}`` log line +per case; ``scripts/ccl_fit.py`` (artifacts) fits the rate and latency. +""" + +import json +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_trace import MESH +from models.demos.deepseek_v3_d_p.tt.v41.ccl import V41Collectives +from models.demos.deepseek_v3_d_p.tt.v41.layout import SP_AXIS, TP_AXIS + +WIDTH = 1024 # bf16 columns per chip: 2 KiB per row +ROWS = (128, 512, 2048, 8192) # 256 KiB .. 16 MiB per chip +LINKS = (1, 2) +ITERS = 8 +REPLAYS = 5 +CASES = ("tp_all_gather", "tp_reduce_scatter", "tp_all_to_all", "sp_all_gather", "sp_all_to_all") + + +def _call(coll: V41Collectives, case: str, x, links: int): + coll.num_links = links + if case == "tp_all_gather": + return coll.tp_all_gather(x, dim=3) + if case == "tp_reduce_scatter": + return coll.tp_reduce_scatter(x, dim=3) + if case == "tp_all_to_all": + return coll.tp_all_to_all(x, in_dim=3, out_dim=2) + if case == "sp_all_gather": + return ttnn.experimental.all_gather_async( + x, + dim=3, + multi_device_global_semaphore=coll.tt_ccl.get_and_cycle_ag_semaphore_handles(cluster_axis=SP_AXIS), + barrier_semaphore=coll.tt_ccl.get_and_cycle_barrier_semaphore_handle(cluster_axis=SP_AXIS), + num_links=links, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + topology=coll.sp_topology, + cluster_axis=SP_AXIS, + ) + if case == "sp_all_to_all": + return ttnn.experimental.all_to_all_async_generic( + x, + in_dim=3, + out_dim=2, + num_links=links, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + topology=coll.sp_topology, + cluster_axis=SP_AXIS, + ) + raise ValueError(case) + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("case", CASES) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_ccl_calibration(mesh_device, device_params, case): + assert tuple(mesh_device.shape) == (2, 4) + coll = V41Collectives(mesh_device) + axis = TP_AXIS if case.startswith("tp") else SP_AXIS + n = mesh_device.shape[axis] + for links in LINKS: + for rows in ROWS: + t0 = time.perf_counter() + x = ttnn.from_torch( + torch.randn(1, 1, rows, WIDTH, dtype=torch.bfloat16), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + y = _call(coll, case, x, links) # compile + correctness of the output shape + want = [1, 1, rows, WIDTH] + if "all_gather" in case: + want[3] *= n + elif "reduce_scatter" in case: + want[3] //= n + else: + want[3], want[2] = WIDTH * n, rows // n + assert list(ttnn.get_device_tensors(y)[0].shape) == want, (case, list(y.shape), want) + ttnn.synchronize_device(mesh_device) + outs = [] + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + for _ in range(ITERS): + outs.append(_call(coll, case, x, links)) + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) # warm replay + ttnn.synchronize_device(mesh_device) + start = time.perf_counter() + for _ in range(REPLAYS): + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + us = (time.perf_counter() - start) * 1e6 / (REPLAYS * ITERS) + ttnn.release_trace(mesh_device, tid) + rec = dict(case=case, axis=axis, chips=n, links=links, rows=rows, in_bytes=rows * WIDTH * 2, us=us) + logger.info(f"CCL_CAL {json.dumps(rec)} (case {time.perf_counter() - t0:.1f}s)") + for t in outs + [x, y]: + ttnn.deallocate(t) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_config.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_config.py new file mode 100644 index 000000000000..99f7ef8ff42d --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_config.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Layer schedule of DeepSeekV41FlashConfig against the hand-written V4.1 table (no device).""" + +import json +from pathlib import Path + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import V41BlockType as T + +VENDORED_CONFIG = Path(__file__).parents[2] / "reference" / "deepseek_v41" / "config.json" + +# discovery.md §2, written out by hand from the six block types. +EXPECTED = { + 0: (T.SWA_ONLY, None, None), + 1: (T.SWA_ONLY, None, None), + 2: (T.KV_INDEX_SOURCE, 2, 2), + 3: (T.CONSUMER_RATIO2, 2, 2), + 7: (T.CONSUMER_RATIO2, 2, 2), + 8: (T.KV_INDEX_SOURCE, 8, 8), + 13: (T.CONSUMER_RATIO2, 8, 8), + 14: (T.KV_INDEX_SOURCE, 14, 14), + 19: (T.CONSUMER_RATIO2, 14, 14), + 20: (T.CANDIDATE_SOURCE, 20, 20), + 21: (T.CONSUMER_RATIO1, 20, 20), + 23: (T.CONSUMER_RATIO1, 20, 20), + 24: (T.CANDIDATE_INDEX_SOURCE, 20, 24), + 27: (T.CONSUMER_RATIO1, 20, 24), + 36: (T.CANDIDATE_INDEX_SOURCE, 20, 36), + 39: (T.CONSUMER_RATIO1, 20, 36), +} + + +def test_block_type_and_sources(expect_error): + for layer, (block_type, kv_source, index_source) in EXPECTED.items(): + assert C.block_type(layer) == block_type, layer + if kv_source is None: + with expect_error(ValueError, "does not attend over compressed KV"): + C.kv_source(layer) + else: + assert C.kv_source(layer) == kv_source, layer + assert C.index_source(layer) == index_source, layer + + +def test_block_type_counts(): + counts = {} + for layer in range(C.NUM_LAYERS): + counts[C.block_type(layer)] = counts.get(C.block_type(layer), 0) + 1 + assert counts == { + T.SWA_ONLY: 2, + T.KV_INDEX_SOURCE: 3, + T.CONSUMER_RATIO2: 15, + T.CANDIDATE_SOURCE: 1, + T.CANDIDATE_INDEX_SOURCE: 4, + T.CONSUMER_RATIO1: 15, + } + + +def test_matches_pinned_reference_config(): + """Constants equal the vendored upstream inference/config.json.""" + cfg = json.loads(VENDORED_CONFIG.read_text()) + pairs = { + "dim": C.EMB_SIZE, + "moe_inter_dim": C.MOE_INTERMEDIATE_SIZE, + "n_layers": C.NUM_LAYERS, + "n_mtp_layers": C.NUM_DSPARK_LAYERS, + "n_heads": C.NUM_ATTENTION_HEADS, + "n_routed_experts": C.NUM_ROUTED_EXPERTS, + "n_activated_experts": C.NUM_EXPERTS_PER_TOKEN, + "q_lora_rank": C.Q_LORA_RANK, + "o_lora_rank": C.O_LORA_RANK, + "o_groups": C.O_GROUPS, + "head_dim": C.HEAD_DIM, + "rope_head_dim": C.QK_ROPE_HEAD_DIM, + "norm_eps": C.RMS_NORM_EPS, + "window_size": C.SLIDING_WINDOW, + "index_n_heads": C.INDEX_N_HEADS, + "index_head_dim": C.INDEX_HEAD_DIM, + "index_topk": C.INDEX_TOPK, + "candidate_source_layer": C.CANDIDATE_SOURCE_LAYER, + "candidate_topk_blocks": C.CANDIDATE_TOPK_BLOCKS, + "candidate_block_size": C.CANDIDATE_BLOCK_SIZE, + "route_scale": C.ROUTE_SCALE, + "swiglu_limit": C.SWIGLU_LIMIT, + "compress_rope_theta": C.COMPRESS_ROPE_THETA, + "rope_factor": C.ROPE_SCALING_FACTOR, + "original_seq_len": C.ROPE_SCALING_ORIGINAL_MAX_POSITION_EMBEDDINGS, + "hc_mult": C.HC_MULT, + "hc_sinkhorn_iters": C.HC_SINKHORN_ITERS, + "hc_eps": C.HC_EPS, + "vocab_size": C.VOCAB_SIZE, + } + for key, value in pairs.items(): + assert cfg[key] == value, key + assert tuple(cfg["compress_ratios"]) == C.COMPRESS_RATIOS + assert tuple(cfg["kv_source_layers"]) == C.KV_SOURCE_LAYERS + assert tuple(cfg["index_source_layers"]) == C.INDEX_SOURCE_LAYERS + assert tuple(cfg["engram_layer_ids"]) == C.ENGRAM_LAYER_IDS + assert tuple(cfg["engram_num_embeddings"]) == C.ENGRAM_NUM_EMBEDDINGS + assert tuple(cfg["dspark_target_layer_ids"]) == C.DSPARK_TARGET_LAYER_IDS diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode.py new file mode 100644 index 000000000000..851cc241d1a4 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode.py @@ -0,0 +1,286 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""V4.1 single-token decode on the Galaxy after a real-weight prefill, against the vendored CPU reference. + +Prefills ``V41_DECODE_PROMPT`` tokens of real text (odd by default, so the ratio-2 compressor carry is +consumed by +the first step) with the layer schedule ``V41_DECODE_LAYERS`` on the device (``TtV41Transformer.prefill``) and +on +the CPU reference (``Transformer.forward(tokens, 0)``), then decodes ``V41_DECODE_STEPS`` tokens step by step +(``TtV41Decoder.decode`` against ``Transformer.forward(token, start_pos)``), teacher-forced on the reference's +greedy +token. Per step it reports the greedy agreement, the logits PCC and every block's output PCC; after the +prefill the +compressor carry against the reference's ``kv_state`` / ``score_state``. With ``V41_DECODE_TRACE=1`` (default) +the +decoder captures the step per position parity after ``V41_DECODE_WARMUP`` eager steps and replays it: each +step +reports its mode and, for replays, the device time of the replay. With ``V41_DECODE_REF=0`` (schedules the +reference cannot run: checkpoint Engram layers) the device decodes alone, teacher-forced on the text's own +next tokens +(``book_top1``: the depth test's next-token accuracy, through the decode step), or free-running on its greedy +token +with ``V41_DECODE_FREE=1``; both report the text. + +With ``V41_DECODE_PROFILE=1`` (under ``python -m tracy -r -p -v -m pytest ...`` with +``TT_METAL_DEVICE_PROFILER=1``) the +device profiler is read after the prefill and after every block of every step, so no step overflows the +device's +marker buffers, and the last step runs between the ``start`` / ``stop`` signposts (its ops are the +measurement). + +Results go to the log as ``V41_DECODE_RESULT {json}`` and to ``V41_DECODE_OUT`` when set. Reported, not gated +here: a NaN or a crash fails the test, a low PCC does not. +""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import model as v41 +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_depth import _dram_bytes, _rss_gb, parse_layers +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.decode import TtV41Decoder +from models.demos.deepseek_v3_d_p.tt.v41.engram_checkpoint import build_engram_modules +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint + +try: + from tracy import signpost +except ModuleNotFoundError: # not under the tracy wrapper + + def signpost(_name: str) -> None: + pass + + +DEFAULT_LAYERS = "0,2,3,20,21,24" +TRACE_REGION = 512 * 1024 * 1024 # as test_v41_trace.py; two captures of the step + + +def _pcc(a: torch.Tensor, b: torch.Tensor) -> float: + a, b = a.flatten().double(), b.flatten().double() + if a.numel() < 2 or torch.isnan(a).any() or torch.isnan(b).any(): + return float("nan") + return float(torch.corrcoef(torch.stack([a, b]))[0, 1]) + + +def _reference_forward(ref: v41.Transformer, ids: torch.Tensor, start_pos: int) -> torch.Tensor: + """``Transformer.forward`` under the bf16 default dtype it was built with (``testing.prefill``) -> fp32 + logits + ``[vocab]`` of the last position.""" + with v41.set_dtype(torch.bfloat16): + _, logits, _ = ref.forward(ids, start_pos) + return logits[0].float() + + +def _streams_to_host(x: ttnn.Tensor, tp: int, hc: int) -> torch.Tensor: + """Row 0 of the decode tile's streams ``[1, 1, 32, hc * hidden/tp]`` (mesh row 0's TP shards) -> ``[hc, + hidden]``.""" + shards = ttnn.get_device_tensors(x) + parts = [ttnn.to_torch(shards[j])[0, 0, 0].float().view(hc, -1) for j in range(tp)] + return torch.cat(parts, dim=-1) + + +@pytest.mark.timeout(int(os.environ.get("V41_DECODE_TIMEOUT", 4 * 3600))) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=TRACE_REGION), indirect=True) +def test_v41_decode(mesh_device, device_params): + layers = parse_layers(os.environ.get("V41_DECODE_LAYERS", DEFAULT_LAYERS)) + prompt = int(os.environ.get("V41_DECODE_PROMPT", 255)) + steps = int(os.environ.get("V41_DECODE_STEPS", 8)) + chunk = int(os.environ.get("V41_DECODE_CHUNK", 1024)) + use_ref = os.environ.get("V41_DECODE_REF", "1") == "1" + free_run = os.environ.get("V41_DECODE_FREE", "0") == "1" + trace = os.environ.get("V41_DECODE_TRACE", "1") == "1" + warmup = int(os.environ.get("V41_DECODE_WARMUP", 4)) + profile = os.environ.get("V41_DECODE_PROFILE", "0") == "1" + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + from transformers import AutoTokenizer + + max_seq = -(-(prompt + steps + 32) // chunk) * chunk + result = { + "layers": layers, + "n_layers": len(layers), + "prompt": prompt, + "steps": steps, + "chunk": chunk, + "max_seq": max_seq, + "reference": use_ref, + "free_run": free_run, + "trace": trace, + "warmup": warmup, + "profile": profile, + "mesh": list(mesh_device.shape), + "fabric": str(device_params.get("fabric_config")), + } + spec = orc.real_spec(tuple(layers), max_seq, checkpoint=ckpt.root) + root = weight_cache_dir(spec, mesh_device.shape) + text = orc.text_tokens(prompt + steps + 1)[0] + tokens, book = text[:prompt], text[prompt:] # the prompt and the text's own continuation + tokenizer = AutoTokenizer.from_pretrained(orc.HF_SNAPSHOT) + sp, tp = tuple(mesh_device.shape) + + ref = None + if use_ref: + t0 = time.time() + ref = orc.build_reference(spec) + orc._reset_state(ref) + result["reference_build_s"] = round(time.time() - t0, 1) + result["rss_after_reference_gb"] = round(_rss_gb(), 1) + logger.info(f"decode: reference built in {result['reference_build_s']} s") + + t0 = time.time() + engram, engram_hash = build_engram_modules(mesh_device, C, ckpt, layers, tokenizer) + top = ckpt.read(["embed.weight", "norm.weight", "head.weight"]) + model = TtV41Transformer( + mesh_device, + C, + layers, + lambda layer, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, layer), + top["embed.weight"], + top["norm.weight"], + top["head.weight"], + max_seq_len=max_seq, + chunk=chunk, + engram=engram, + engram_hash=engram_hash, + weight_cache_path=root, + ) + decoder = TtV41Decoder(model, root, trace=trace, warmup=warmup) + result["build_s"] = round(time.time() - t0, 1) + result["dram_after_build_bytes"] = _dram_bytes(mesh_device) + + # --- prefill on both --- + t0 = time.time() + logits, state = model.prefill(tokens, logit_positions=1) + result["prefill_s"] = round(time.time() - t0, 3) + dev_logits = logits[-1] + assert not torch.isnan(dev_logits).any(), "NaN in the device prefill logits" + if profile: + ttnn.ReadDeviceProfiler(mesh_device) + decoder.begin(tokens, state) + if ref is not None: + t0 = time.time() + ref_logits = _reference_forward(ref, tokens[None], 0) + result["reference_prefill_s"] = round(time.time() - t0, 1) + result["prefill_logits_pcc"] = round(_pcc(dev_logits, ref_logits), 5) + result["prefill_argmax"] = [int(dev_logits.argmax()), int(ref_logits.argmax())] + carry = {} + for pos, layer in enumerate(layers): + if state.compressor_carry.get(layer) is not None: + comp = ref.layers[pos].attn.compressor + kv, score = (state.to_host(t)[0] for t in state.compressor_carry[layer]) + slot = (prompt - 1) % comp.compress_ratio + carry[layer] = { + "kv": round(_pcc(kv, comp.kv_state[0, slot]), 5), + "score": round(_pcc(score, comp.score_state[0, slot]), 5), + } + result["prefill_carry_pcc"] = carry + logger.info(f"decode: prefill logits PCC {result['prefill_logits_pcc']}, carry {carry}") + + # --- decode --- + captured = {} + ref_blocks = {} + hooks = [] + if ref is not None: + # every block's output to the host, for the per-block PCC: 40 reads per step, only worth it with a + # reference + decoder.on_block = lambda layer, x: captured.__setitem__(layer, _streams_to_host(x, tp, C.HC_MULT)) + for pos, layer in enumerate(layers): + hooks.append( + ref.layers[pos].register_forward_hook( + lambda mod, args, out, layer=layer: ref_blocks.__setitem__(layer, out[0][0, 0].float()) + ) + ) + if profile: + previous = decoder.on_block + + def drain(layer, x): + if previous is not None: + previous(layer, x) + ttnn.ReadDeviceProfiler(mesh_device) + + decoder.on_block = drain + token = int((ref_logits if ref is not None else dev_logits).argmax()) + if ref is None and not free_run: + token = int(book[0]) # teacher forcing on the text + generated, per_step = [token], [] + try: + for i in range(steps): + p = state.start + captured.clear() + if profile and i == steps - 1: + signpost("start") + t0 = time.time() + dev_logits = decoder.decode(token, state) + wall = time.time() - t0 + if profile: + ttnn.ReadDeviceProfiler(mesh_device) + if i == steps - 1: + signpost("stop") + assert not torch.isnan(dev_logits).any(), f"NaN in the device logits at position {p}" + entry = {"pos": p, "token": token, "dev_argmax": int(dev_logits.argmax()), "wall_s": round(wall, 3)} + entry.update(decoder.last) + entry["book_next"] = int(book[i + 1]) + if ref is not None: + t0 = time.time() + ref_logits = _reference_forward(ref, torch.tensor([[token]]), p) + entry["ref_s"] = round(time.time() - t0, 2) + entry["ref_argmax"] = int(ref_logits.argmax()) + entry["logits_pcc"] = round(_pcc(dev_logits, ref_logits), 5) + entry["top5_hit"] = entry["dev_argmax"] in ref_logits.topk(5).indices.tolist() + entry["block_pcc"] = { + str(layer): round(_pcc(captured[layer], ref_blocks[layer]), 5) for layer in layers + } + token = entry["ref_argmax"] # teacher forcing on the reference + else: + token = entry["dev_argmax"] if free_run else entry["book_next"] + generated.append(token) + per_step.append(entry) + logger.info(f"decode step {i} at {p}: {entry}") + finally: + for h in hooks: + h.remove() + result["steps_detail"] = per_step + replays = [e for e in per_step if e["mode"] == "replay"] + result["replay_steps"] = len(replays) + if replays: + result["trace_segments"] = decoder.trace_segments + result["replay_wall_s"] = [e["wall_s"] for e in replays] + result["replay_wall_min_s"] = min(e["wall_s"] for e in replays) + result["replay_device_s"] = [e["device_s"] for e in replays] + result["replay_device_min_s"] = min(e["device_s"] for e in replays) + for key in ("inputs_s", "stage_s", "sync_s", "issue_s", "read_s", "history_s"): + result["replay_" + key] = [e.get(key) for e in replays] + # the captured graphs alone, replayed back to back without host work: the device's time per token + result["replay_device_only_s"] = decoder.replay_device_time() + decoder.release() + result["fed"] = generated # the tokens fed in (teacher-forced or the device's own) + result["text"] = tokenizer.decode([e["dev_argmax"] for e in per_step]) # the device's greedy predictions + result["book_text"] = tokenizer.decode(book[1 : steps + 1].tolist()) + result["book_top1"] = round(sum(e["dev_argmax"] == e["book_next"] for e in per_step) / len(per_step), 4) + result["step_wall_s"] = [e["wall_s"] for e in per_step] + result["step_wall_min_s"] = min(e["wall_s"] for e in per_step) + if ref is not None: + result["top1_agreement"] = round(sum(e["dev_argmax"] == e["ref_argmax"] for e in per_step) / len(per_step), 4) + result["top5_agreement"] = round(sum(e["top5_hit"] for e in per_step) / len(per_step), 4) + result["logits_pcc_min"] = min(e["logits_pcc"] for e in per_step) + result["block_pcc_min"] = {str(layer): min(e["block_pcc"][str(layer)] for e in per_step) for layer in layers} + result["dram_after_decode_bytes"] = _dram_bytes(mesh_device) + result["rss_peak_gb"] = round(_rss_gb(), 1) + logger.info("V41_DECODE_RESULT " + json.dumps(result)) + out = os.environ.get("V41_DECODE_OUT") + if out: + with open(out, "w") as f: + json.dump(result, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_capacity.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_capacity.py new file mode 100644 index 000000000000..f2942f772f57 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_capacity.py @@ -0,0 +1,372 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The batched decode step at long pool capacities: the whole model, 32 users joined from short prompts, then +the +pools filled with synthetic keys up to the capacity and the users moved to positions near it, so the step's +indexer (the dense scores, the candidate selection, the candidate-row scores), sparse attention and writes run +at +``V41_CAP_SEQ`` of context. Timing scope only: the keys are not a real history, so the text is meaningless; +the +correctness of the selection is ``test_indexer.py::test_v41_decode_selection`` and of the step +``test_v41_batch_decode.py``. + +Records (``V41_CAP_RESULT``, ``V41_CAP_OUT``): per step the device time of the replayed capture, the +allocator's +DRAM after the build, the join, the fill, the capture and the replays, the trace region's use, and, with +``V41_CAP_GRAPH=1``, one eager step under ``ttnn.graph`` capture: every op whose inputs or outputs hold at +least +``V41_CAP_GRAPH_MB`` (64) MB, so a conversion of the history shows up by name. + +Phases: the pool's capacity (``V41_CAP_SEQ``), the live users' context (``V41_CAP_POSITIONS``, +a comma list of positions; default ``cap - V41_CAP_MARGIN``) and the number of live rows (``V41_CAP_LIVE``, a +comma +list of slot counts; default every slot) are independent. The 32-row step is warmed up and captured with every +slot +stepping (the served order), then for each position and each live count the first ``b`` slots (slot 0 on, no +rotation: the served c1 shape at ``b = 1``) step ``V41_CAP_STEPS`` replays from that position, every other +slot +idle (a zero row that writes nothing). Each phase records the replays' device time (sorted), the wall and the +host +split (``inputs_s``, ``stage_s``, ``read_s``) as medians, and the capture replayed back to back without host +work +(``replay_device_only_s``), under ``phases``; the first phase's position and live count are the top-level +fields. +``V41_CAP_PROFILE=1``: under ``TT_METAL_DEVICE_PROFILER=1`` with the C++ post-process and +mid-run dumps, the profiler is read once after the joins (whose markers overflow the buffer and are dropped), +after the fill and after every step, so a step's device buffers never overflow and the replays' per-op rows +land +in ``cpp_device_perf_report.csv`` by trace id. + +A production session is ``V41_CAP_SEQ=1054720 V41_CAP_CHUNK=5120`` (the served ``max_seq_len`` and chunk) with +the catalog's flags; the prefill traces the server captures at warm-up are not captured here (eager prefills +of +short prompts join the slots), so the trace region holds the decode capture alone. +""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_depth import _dram_bytes, _rss_gb, parse_layers +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.batch_decode import TtV41BatchDecoder +from models.demos.deepseek_v3_d_p.tt.v41.engram_checkpoint import build_engram_modules +from models.demos.deepseek_v3_d_p.tt.v41.pool import BLOCK_ROWS +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint + +TRACE_REGION = 2 * 1024**3 +FILL_ROWS = 4096 # rows of the random block the pools are filled from + + +def _trace_bytes(mesh_device) -> int | None: + try: + view = ttnn.get_memory_view(mesh_device, ttnn.BufferType.TRACE) + return int(view.total_bytes_allocated_per_bank * view.num_banks) + except Exception as e: + logger.warning(f"trace statistics unavailable: {e}") + return None + + +def _fill(mesh_device, dst, block_rows: int, g, layout) -> int: + """Overwrite every row of ``dst`` ([1, 1, H, W] or [U, 1, H, W], a multiple of ``block_rows`` tall) with + copies + of one random bf16 block of ``block_rows`` rows (uploaded once; ``slice_write`` over the rest) -> + writes.""" + shape = list(dst.shape) + w = shape[3] + block = ttnn.from_torch( + (torch.randn(1, 1, block_rows, w, generator=g) * 0.5).to(torch.bfloat16), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=layout, + mesh_mapper=ttnn.ReplicateTensorToMesh(mesh_device), + ) + writes = 0 + for u in range(shape[0]): + for first in range(0, shape[2], block_rows): + n = min(block_rows, shape[2] - first) + piece = block if n == block_rows else ttnn.slice(block, [0, 0, 0, 0], [1, 1, n, w]) + ttnn.experimental.slice_write(piece, dst, [u, 0, first, 0], [u + 1, 1, first + n, w], [1, 1, 1, 1]) + writes += 1 + ttnn.deallocate(block) + return writes + + +@pytest.mark.timeout(int(os.environ.get("V41_CAP_TIMEOUT", 3 * 3600))) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=TRACE_REGION), indirect=True) +def test_v41_decode_capacity(mesh_device, device_params): + layers = parse_layers(os.environ.get("V41_CAP_LAYERS", "all")) + cap = int(os.environ.get("V41_CAP_SEQ", 1048576)) # the pool's positions per user + steps = int(os.environ.get("V41_CAP_STEPS", 12)) + chunk = int(os.environ.get("V41_CAP_CHUNK", 1024)) + users = int(os.environ.get("V41_CAP_USERS", 4)) + base = int(os.environ.get("V41_CAP_PROMPT", 255)) + warmup = int(os.environ.get("V41_CAP_WARMUP", 2)) + margin = int(os.environ.get("V41_CAP_MARGIN", 64)) # the users start this far below the capacity: steps + slots + fill = os.environ.get("V41_CAP_FILL", "1") == "1" + graph = os.environ.get("V41_CAP_GRAPH", "0") == "1" + graph_mb = int(os.environ.get("V41_CAP_GRAPH_MB", 64)) + live_counts = [int(b) for b in os.environ.get("V41_CAP_LIVE", "").split(",") if b] # per phase; [] = all slots + position_list = [int(p) for p in os.environ.get("V41_CAP_POSITIONS", "").split(",") if p] # [] = cap - margin + # V41_CAP_PROFILE=1: under the device profiler, read the profiler once after the joins + # (their markers overflow the buffer and are dropped: a drain processes ~5 minutes of markers a join), + # after + # the fill and after every step, so a step's buffers never overflow and (with + # TT_METAL_PROFILER_MID_RUN_DUMP=1) the host's marker set stays bounded; the replays are told apart in the + # C++ report by their trace columns + profile = os.environ.get("V41_CAP_PROFILE", "0") == "1" + + def drain() -> None: + if profile: + ttnn.ReadDeviceProfiler(mesh_device) + + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + from transformers import AutoTokenizer + + sp, tp = tuple(mesh_device.shape) + n_slots = sp * users + lengths = [base - (s * 5) % 32 for s in range(n_slots)] + stride = base + steps + 8 + text = orc.text_tokens(n_slots * stride)[0] + prompts = {s: text[s * stride : s * stride + lengths[s]] for s in range(n_slots)} + books = {s: text[s * stride + lengths[s] : s * stride + lengths[s] + steps + 1] for s in range(n_slots)} + max_seq = -(-(base + steps + 32) // chunk) * chunk # the model's own states: short + assert cap % chunk == 0 and cap >= max_seq, (cap, chunk, max_seq) + result = { + "layers": layers, + "n_layers": len(layers), + "slots": n_slots, + "users_per_rank": users, + "capacity": cap, + "chunk": chunk, + "model_max_seq": max_seq, + "steps": steps, + "warmup": warmup, + "fill": fill, + "mesh": [sp, tp], + "fabric": str(device_params.get("fabric_config")), + "live_counts": live_counts or [n_slots], + "position_list": position_list or [cap - margin], + "prompt_length": base, + } + assert all(0 < b <= n_slots for b in live_counts), (live_counts, n_slots) + assert all(C.SLIDING_WINDOW + n_slots <= p and p + steps + n_slots <= cap for p in position_list), ( + position_list, + cap, + ) + spec = orc.real_spec(tuple(layers), max_seq, checkpoint=ckpt.root) + root = weight_cache_dir(spec, mesh_device.shape) + tokenizer = AutoTokenizer.from_pretrained(orc.HF_SNAPSHOT) + + t0 = time.time() + engram, engram_hash = build_engram_modules(mesh_device, C, ckpt, layers, tokenizer) + top = ckpt.read(["embed.weight", "norm.weight", "head.weight"]) + model = TtV41Transformer( + mesh_device, + C, + layers, + lambda layer, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, layer), + top["embed.weight"], + top["norm.weight"], + top["head.weight"], + max_seq_len=max_seq, + chunk=chunk, + engram=engram, + engram_hash=engram_hash, + weight_cache_path=root, + ) + result["dram_after_model_bytes"] = _dram_bytes(mesh_device) + decoder = TtV41BatchDecoder( + model, root, users_per_rank=users, trace=True, warmup=warmup, sample_on_device=True, max_seq_len=cap + ) + result["build_s"] = round(time.time() - t0, 1) + result["dram_after_build_bytes"] = _dram_bytes(mesh_device) + pool = decoder.pool + result["pool"] = { + str(src): {"cap": slot.cap, "rows": slot.rows, "layers": list(slot.layers)} for src, slot in pool.slots.items() + } + + t0 = time.time() + for s in range(n_slots): + _, state = model.prefill(prompts[s], logit_positions=1) + decoder.join(s, prompts[s], state) + for group in (state.kv, state.index_k, state.window_carry): + for t in group.values(): + ttnn.deallocate(t) + if state.swa_scratch is not None: + ttnn.deallocate(state.swa_scratch) + for carry in state.carry_buffers.values(): + for t in carry: + ttnn.deallocate(t) + drain() # once: a join overflows the buffer and drops its own markers, and a drain costs minutes of processing + result["prefill_join_s"] = round(time.time() - t0, 1) + result["dram_after_join_bytes"] = _dram_bytes(mesh_device) + + # --- the synthetic history: random keys over every pool, the users at positions near the capacity --- + if fill: + t0 = time.time() + g = torch.Generator().manual_seed(cap) + writes = 0 + for src, t in pool.index_k.items(): + writes += _fill(mesh_device, t, FILL_ROWS, g, ttnn.TILE_LAYOUT) + for src, t in pool.index_k_blocks.items(): + writes += _fill(mesh_device, t, FILL_ROWS // BLOCK_ROWS, g, ttnn.ROW_MAJOR_LAYOUT) + ttnn.synchronize_device(mesh_device) + drain() + result["fill_s"] = round(time.time() - t0, 1) + result["fill_writes"] = writes + # (the compressed KV rows stay as the join left them: zeros past the prompt; the attention reads the + # same + # bytes either way) + + def place(top: int) -> dict: + """Every slot ``top - s``: near ``top``, both parities, distinct; the step writes at position // ratio + and + must stay below cap.""" + positions = {s: top - s for s in range(n_slots)} + for s in range(n_slots): + decoder.slots[s].position = positions[s] + return positions + + positions = place(result["position_list"][0]) + result["positions"] = {str(s): positions[s] for s in (0, n_slots - 1)} + result["live"] = result["live_counts"][0] + result["position"] = result["position_list"][0] + result["dram_after_fill_bytes"] = _dram_bytes(mesh_device) + + # --- the steps: warm-up eager, one capture, replays (every slot stepping: the served capture order) --- + graph_pending = [graph] # the session's first step runs under the graph capture + + def run_steps(n: int, stepping: list, tag: str) -> list: + out = [] + for i in range(n): + tokens = {s: int(books[s][i % steps]) for s in stepping} + capture_graph, graph_pending[0] = graph_pending[0], False + t0 = time.time() + if capture_graph: + ttnn.graph.begin_graph_capture(ttnn.graph.RunMode.NORMAL) + decoder.decode(tokens, {s: (1, 0.0, 0.0) for s in tokens}) + if capture_graph: + captured = ttnn.graph.end_graph_capture() + result["graph_big_ops"] = _big_ops(captured, graph_mb << 20) + result["graph_ops"] = sum(1 for node in captured if node.get("node_type") == "function_start") + wall = time.time() - t0 + entry = {"step": i, "wall_s": round(wall, 3), "mode": decoder.last.get("mode"), "live": len(stepping)} + entry.update({k: round(v, 4) for k, v in decoder.last.items() if isinstance(v, float)}) + if entry["mode"] == "capture": + result["dram_after_capture_bytes"] = _dram_bytes(mesh_device) + result["trace_after_capture_bytes"] = _trace_bytes(mesh_device) + out.append(entry) + logger.info(f"capacity step {i} ({tag}): {json.dumps(entry)}") + drain() + return out + + def summarise(entries: list) -> dict: + replays = [e for e in entries if e["mode"] == "replay"] + out = {"replay_steps": len(replays)} + if replays: + out["replay_wall_min_s"] = min(e["wall_s"] for e in replays) + out["replay_wall_median_s"] = sorted(e["wall_s"] for e in replays)[len(replays) // 2] + out["replay_device_s"] = sorted(e["device_s"] for e in replays) + out["replay_device_min_s"] = out["replay_device_s"][0] + out["replay_device_median_s"] = out["replay_device_s"][len(replays) // 2] + for key in ("inputs_s", "stage_s", "sync_s", "issue_s", "read_s"): + vals = sorted(e[key] for e in replays if key in e) + if vals: + out[key + "_median"] = vals[len(vals) // 2] + return out + + all_slots = list(range(n_slots)) + + def phase(top: int, stepping: list, detail: list) -> dict: + out = {"position": top, "live": len(stepping), "slots": [stepping[0], stepping[-1]], "steps_detail": detail} + out.update(summarise(detail)) + if out["replay_steps"]: # the capture replayed on the phase's last staged inputs, no host work + out["replay_device_only_s"] = decoder.replay_device_time(repeats=7) + drain() + logger.info( + f"capacity phase position {top} live {len(stepping)}: " + f"{json.dumps({k: v for k, v in out.items() if k != 'steps_detail'})}" + ) + return out + + phases = [] + if not live_counts and not position_list: # the original single phase: every slot, from cap - margin + phases.append(phase(positions[0], all_slots, run_steps(steps, all_slots, "steps"))) + else: + warm = run_steps(warmup + 1, all_slots, "warmup") # eager x warmup, then the capture: the served order + assert decoder.capture is not None, "the 32-row step was not captured" + result["warmup_detail"] = warm + for top in result["position_list"]: + for b in result["live_counts"]: + place(top) + stepping = all_slots[:b] + phases.append(phase(top, stepping, run_steps(steps, stepping, f"pos {top} live {b}"))) + result["phases"] = phases + result["steps_detail"] = [e for ph in phases for e in ph["steps_detail"]] + first = phases[0] + result["replay_steps"] = first["replay_steps"] + if first["replay_steps"]: + for key in ("replay_wall_min_s", "replay_device_s", "replay_device_min_s", "replay_device_median_s"): + result[key] = first[key] + result["replay_device_only_s"] = first["replay_device_only_s"] + result["dram_after_decode_bytes"] = _dram_bytes(mesh_device) + result["trace_after_decode_bytes"] = _trace_bytes(mesh_device) + decoder.release() + result["rss_peak_gb"] = round(_rss_gb(), 1) + logger.info("V41_CAP_RESULT " + json.dumps(result)) + out_path = os.environ.get("V41_CAP_OUT") + if out_path: + with open(out_path, "w") as f: + json.dump(result, f, indent=1) + + +def _tensor_bytes(info) -> int: + """Bytes of a captured graph node's tensor description (shape x dtype), 0 if unknown.""" + try: + shape = info.get("shape") or info.get("params", {}).get("shape") + if isinstance(shape, str): + shape = [int(x) for x in shape.strip("[]()").replace("Shape", "").strip("()[] ").split(",") if x.strip()] + n = 1 + for d in shape: + n *= int(d) + return n * 2 + except Exception: + return 0 + + +def _big_ops(captured, threshold: int) -> list: + """The captured graph's ops (``function_start`` nodes) with any tensor of at least ``threshold`` bytes + among + their arguments, as (name, [bytes...]): a conversion of a pool shows up here.""" + out = [] + for node in captured: + if node.get("node_type") != "function_start": + continue + name = node.get("params", {}).get("name", "?") + sizes = [] + for arg in node.get("arguments", []): + if isinstance(arg, dict): + b = _tensor_bytes(arg) + if b >= threshold: + sizes.append(b) + elif isinstance(arg, str) and "Shape" in arg: + b = _tensor_bytes({"shape": arg[arg.index("Shape") :].split(")")[0]}) + if b >= threshold: + sizes.append(b) + if sizes: + out.append([name, sizes]) + return out diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_dense.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_dense.py new file mode 100644 index 000000000000..a3a31f6d62f5 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_dense.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The decode tile's dense projections as one DRAM-rate program (``tt/v41/decode_dense.py``, +``V41_DECODE_DENSE=matvec``) against ``ttnn.linear`` on the served compute config and against an fp32 +reference of the +dequantized weight, on the attention's five projection shapes (``wq_a``, ``wkv``, ``wq_b``, ``wo_a`` with its +two +groups, ``wo_b``) at TP 4: random bf16 rows (every row of the tile, every chip its own), the distance of both +paths +to the reference, then each under trace (``V41_DD_REPLAYS`` replays of ``V41_DD_CALLS`` calls) for the +per-call +device time. No weights; ~1 minute on the Galaxy. Results: ``V41_DD_RESULT``, ``V41_DD_OUT``.""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.v41.attention import ( + DECODE_1D, + DENSE_COMPUTE_CONFIG, + DENSE_IN0_BLOCK_W, + dense_program_config, +) +from models.demos.deepseek_v3_d_p.tt.v41.decode_dense import DecodeDense, column_memory_config + +TP = 4 +# name -> (B, K, N) per chip at TP 4 +SHAPES = { + "wq_a": (1, C.EMB_SIZE // TP, C.Q_LORA_RANK), + "wkv": (1, C.EMB_SIZE // TP, C.HEAD_DIM + C.QK_ROPE_HEAD_DIM), + "wq_b": (1, C.Q_LORA_RANK, C.NUM_ATTENTION_HEADS // TP * C.HEAD_DIM), + "wo_a": (C.O_GROUPS // TP, C.NUM_ATTENTION_HEADS * C.HEAD_DIM // C.O_GROUPS, C.O_LORA_RANK), + "wo_b": (1, C.O_GROUPS * C.O_LORA_RANK // TP, C.EMB_SIZE), +} + + +def _trace_us(mesh_device, fn, calls: int, replays: int) -> float: + for _ in range(2): + ttnn.deallocate(fn()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn() for _ in range(calls)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / calls + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + return round(best * 1e6, 1) + + +def _stats(a: torch.Tensor, b: torch.Tensor) -> dict: + a, b = a.double().flatten(), b.double().flatten() + pcc = torch.corrcoef(torch.stack([a, b]))[0, 1].item() + rel = ((a - b).norm() / b.norm().clamp_min(1e-30)).item() + return {"pcc": round(pcc, 7), "rel": round(rel, 6), "max_abs": round((a - b).abs().max().item(), 5)} + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=64 * 1024 * 1024), indirect=True) +def test_v41_decode_dense(mesh_device, device_params): + replays = int(os.environ.get("V41_DD_REPLAYS", 20)) + calls = int(os.environ.get("V41_DD_CALLS", 32)) + sp, tp = tuple(mesh_device.shape) + chips = sp * tp + torch.manual_seed(130) + result = {"mesh": list(mesh_device.shape), "shapes": {}} + replicate = ttnn.ReplicateTensorToMesh(mesh_device) + shard = ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, 1)) + for name, (B, K, N) in SHAPES.items(): + w_host = torch.randn(1, B, K, N) * 0.02 + # the bfp8 rounding is the host's (``from_torch`` without a device) for both layouts: ``from_torch`` + # with a + # device and a sharded memory config rounds differently (1 ulp on a tenth of the words) + w_h = ttnn.from_torch(w_host, dtype=ttnn.bfloat8_b, layout=ttnn.TILE_LAYOUT, mesh_mapper=replicate) + w = ttnn.to_device(w_h, mesh_device) + w_deq = ttnn.to_torch(ttnn.get_device_tensors(w)[0]).float() # the bfp8 weight as the device holds it + w_col = ttnn.to_device(w_h, mesh_device, memory_config=column_memory_config(mesh_device, K)) + assert torch.equal( + ttnn.to_torch(ttnn.get_device_tensors(w_col)[0]), ttnn.to_torch(ttnn.get_device_tensors(w)[0]) + ) + x_host = torch.randn(sp, tp * B, 32, K).to(torch.bfloat16) # chip (r, c): its B row blocks + x = ttnn.from_torch(x_host, device=mesh_device, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, mesh_mapper=shard) + x_host = x_host.view(chips, B, 32, K) + dense = DecodeDense(w, mesh_device) + dense_col = DecodeDense(w_col, mesh_device) + assert dense_col.col_contig and not dense.col_contig + + def linear(): + if name in DENSE_IN0_BLOCK_W: + cfg = dense_program_config(32, K, N, DENSE_IN0_BLOCK_W[name], one_d=name in DECODE_1D) + return ttnn.linear(x, w, program_config=cfg, compute_kernel_config=DENSE_COMPUTE_CONFIG) + return ttnn.linear(x, w, compute_kernel_config=DENSE_COMPUTE_CONFIG) + + ref = torch.matmul(x_host.float(), w_deq) # [chips, B, 32, N] + got_lin = torch.cat([ttnn.to_torch(t).float() for t in ttnn.get_device_tensors(linear())]) + got_mv = torch.cat([ttnn.to_torch(t).float() for t in ttnn.get_device_tensors(dense(x))]) + assert got_mv.shape == ref.shape, (got_mv.shape, ref.shape) + got_col = torch.cat([ttnn.to_torch(t).float() for t in ttnn.get_device_tensors(dense_col(x))]) + assert torch.equal(got_col, got_mv), name # the same program on the same bytes: bit for bit + entry = { + "B": B, + "K": K, + "N": N, + "cols": dense.cols, + "S": dense.S, + "L": dense.L, + "blk": dense.BLK, + "units_max": dense.units_max, + "linear_vs_ref": _stats(got_lin, ref), + "matvec_vs_ref": _stats(got_mv, ref), + "matvec_vs_linear": _stats(got_mv, got_lin), + "mb": round(B * K * N * 1088 / 1024 / 1e6, 2), + } + # the timing under trace + entry["linear_us"] = _trace_us(mesh_device, linear, calls, replays) + entry["matvec_us"] = _trace_us(mesh_device, lambda: dense(x), calls, replays) + entry["matvec_col_us"] = _trace_us(mesh_device, lambda: dense_col(x), calls, replays) + for k in ("linear", "matvec", "matvec_col"): + entry[f"{k}_gbps"] = round(entry["mb"] / entry[f"{k}_us"] * 1e3, 1) + # the prefill's matmul on the column-sharded weight (the layout can replace the served one only if + # ttnn.linear reads it on the chunk shapes no slower): 640 rows a chip (5,120 / sp) on the 2D config + rows = 5120 // sp + xp_host = torch.randn(sp, tp * B, rows, K).to(torch.bfloat16) + xp = ttnn.from_torch( + xp_host, device=mesh_device, dtype=ttnn.bfloat16, layout=ttnn.TILE_LAYOUT, mesh_mapper=shard + ) + cfg = dense_program_config(rows, K, N, DENSE_IN0_BLOCK_W.get(name, 8)) + + def plinear(wt): + return ttnn.linear(xp, wt, program_config=cfg, compute_kernel_config=DENSE_COMPUTE_CONFIG) + + try: + p_int = torch.cat([ttnn.to_torch(t).float() for t in ttnn.get_device_tensors(plinear(w))]) + p_col = torch.cat([ttnn.to_torch(t).float() for t in ttnn.get_device_tensors(plinear(w_col))]) + entry["prefill_col_equal"] = bool(torch.equal(p_int, p_col)) + entry["prefill_linear_us"] = _trace_us(mesh_device, lambda: plinear(w), 4, replays) + entry["prefill_linear_col_us"] = _trace_us(mesh_device, lambda: plinear(w_col), 4, replays) + except Exception as e: # noqa: BLE001 + entry["prefill_col_error"] = str(e).splitlines()[0][:200] + ttnn.deallocate(xp) + logger.info(f"decode dense {name}: {entry}") + result["shapes"][name] = entry + assert entry["matvec_vs_ref"]["pcc"] > 0.9999, (name, entry) + assert entry["matvec_vs_ref"]["rel"] <= entry["linear_vs_ref"]["rel"] * 1.5 + 1e-6, (name, entry) + ttnn.deallocate(w) + ttnn.deallocate(w_col) + ttnn.deallocate(x) + logger.info(f"V41_DD_RESULT {json.dumps(result)}") + out = os.environ.get("V41_DD_OUT") + if out: + json.dump(result, open(out, "w"), indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_inputs.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_inputs.py new file mode 100644 index 000000000000..479abf259cb7 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_inputs.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The batched decode step's packed host inputs (``V41_DECODE_HOST_INPUTS=packed``): the RoPE +rows from host tables equal ``cos_sin`` at the same positions bit for bit; ``StepPacker.pack`` + ``unpack`` on +the +mesh gives every chip the tensors the per-input path gave it (shapes, dtypes, bits); an Engram layer's packed +shards with ``live_rows`` equal the full path's shards on the chips that hold a live row and the zero shard on +the +others. No weights; ~20 s on the Galaxy.""" + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.v41.batch_decode import StepPacker +from models.demos.deepseek_v3_d_p.tt.v41.decode import ROWS +from models.demos.deepseek_v3_d_p.tt.v41.engram import PACKED_WIDTH, TtV41Engram, V41EngramTable +from models.demos.deepseek_v3_d_p.tt.v41.rope import RopeTables, cos_sin + + +def test_v41_rope_tables(): + """``RopeTables.rows`` against ``cos_sin`` (bf16) at random positions, the chunk boundaries and the top.""" + length = 70000 + tables = RopeTables(C, length, chunk=65536) + g = torch.Generator().manual_seed(59) + positions = torch.cat( + [torch.randint(0, length, (512,), generator=g), torch.tensor([0, 1, 65535, 65536, 65537, length - 1])] + ) + for compressed in (False, True): + cos, sin = tables.rows(compressed, positions) + ref_cos, ref_sin = (t.view(len(positions), -1).to(torch.bfloat16) for t in cos_sin(C, compressed, positions)) + assert cos.dtype == torch.bfloat16 and cos.shape == (len(positions), C.QK_ROPE_HEAD_DIM) + assert torch.equal(cos.view(torch.int16), ref_cos.view(torch.int16)), compressed + assert torch.equal(sin.view(torch.int16), ref_sin.view(torch.int16)), compressed + + +class _Pool: + """``V41DecodePool.per_rank`` alone.""" + + def __init__(self, mesh_device): + self.mesh_device = mesh_device + self.sp, self.tp = tuple(mesh_device.shape) + + def per_rank(self, host, dtype, layout=ttnn.ROW_MAJOR_LAYOUT, device=True): + assert host.shape[0] == self.sp, (tuple(host.shape), self.sp) + return ttnn.from_torch( + host, + device=self.mesh_device if device else None, + dtype=dtype, + layout=layout, + mesh_mapper=ttnn.ShardTensor2dMesh(self.mesh_device, (self.sp, self.tp), dims=(0, None)), + ) + + +def _chips(t) -> list[torch.Tensor]: + return [ttnn.to_torch(d) for d in ttnn.get_device_tensors(t)] + + +@pytest.mark.parametrize("mode", ["packed", "one"]) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(), indirect=True) +def test_v41_step_packer(mesh_device, device_params, mode): + """pack -> upload -> unpack on the mesh against the per-input path (``from_torch`` of each value); + ``mode`` + ``packed`` (the three or four host tensors) or ``one`` (one uint32 tensor with the bf16 + rows as fp32 bit patterns, bitcast and tilized on device). A second pack of other values + into the same packer must read back those values (the host rows are reused).""" + pool = _Pool(mesh_device) + sp, tp = pool.sp, pool.tp + g = torch.Generator().manual_seed(59) + rope_dim = C.QK_ROPE_HEAD_DIM + ints = { + "token": torch.randint(0, C.VOCAB_SIZE, (sp, ROWS), generator=g, dtype=torch.int32), + "k": torch.randint(1, 33, (sp, ROWS), generator=g, dtype=torch.int32), + "ring": torch.full((sp, ROWS), 77, dtype=torch.int32), + "newest1": torch.randint(-1, 1000, (sp, ROWS), generator=g, dtype=torch.int32), + "comp1": torch.randint(-1, 1000, (sp, ROWS), generator=g, dtype=torch.int32), + "newest2": torch.randint(-1, 1000, (sp, ROWS), generator=g, dtype=torch.int32), + "comp2": torch.randint(-1, 1000, (sp, ROWS), generator=g, dtype=torch.int32), + } + floats = {"p": torch.rand(sp, ROWS, generator=g), "temp": torch.rand(sp, ROWS, generator=g) * 3} + rope = { + k: tuple(torch.randn(sp * ROWS, rope_dim, generator=g).to(torch.bfloat16) for _ in range(2)) + for k in ("rope0", "ropec", "ropeg") + } + visible = {"visible1": torch.randint(0, 1000, (sp, ROWS), generator=g).float()} + visible["visible2"] = (visible["visible1"] + 1) // 2 + + packer = StepPacker(pool, sp, mode) + # the first pack holds other values (its upload allocates the buffers); the second, of the values above, + # must + # land in the same rows and columns (the capture reads fixed slices) and replace the first's + other_ints = {k: v + 1 for k, v in ints.items()} + other_rope = {k: tuple(t + 1 for t in v) for k, v in rope.items()} + host = packer.pack(other_ints, floats, other_rope, visible) + if mode == "one": + assert set(host) == {StepPacker.ONE, StepPacker.VISIBLE} + else: + assert set(host) == {StepPacker.INTS, StepPacker.FLOATS, StepPacker.ROPE, StepPacker.VISIBLE} + buffers = {k: ttnn.to_device(v, mesh_device) for k, v in host.items()} + host2 = packer.pack(ints, floats, rope, visible) + for k, v in host2.items(): + ttnn.copy_host_to_device_tensor(v, buffers[k]) + out = packer.unpack(buffers) + + def expect(k, ref, dtype, layout, shape): + got = out[k] + assert got.dtype == dtype and got.layout == layout, (k, got.dtype, got.layout) + assert tuple(got.shape) == shape, (k, tuple(got.shape), shape) + for chip, (mine, theirs) in enumerate(zip(_chips(got), _chips(ref))): + assert mine.shape == theirs.shape, (k, chip, mine.shape, theirs.shape) + if mine.dtype.is_floating_point: + mine, theirs = mine.view(torch.int16), theirs.view(torch.int16) + assert torch.equal(mine, theirs), (k, chip) + + def flat(value, dtype): # the sampler's per-row tensors: [sp * ROWS] sharded over the ranks + shard = ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, None)) + return ttnn.from_torch(value.reshape(-1), dtype=dtype, layout=ttnn.ROW_MAJOR_LAYOUT, mesh_mapper=shard) + + for k, v in ints.items(): + if k == "k": + expect(k, flat(v, ttnn.uint32), ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (ROWS,)) + else: + ref = pool.per_rank(v.view(sp, 1, 1, ROWS), ttnn.uint32, device=False) + expect(k, ref, ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 1, ROWS)) + for k, v in floats.items(): + expect(k, flat(v, ttnn.bfloat16), ttnn.bfloat16, ttnn.ROW_MAJOR_LAYOUT, (ROWS,)) + for k, (cos, sin) in rope.items(): + got = out[k] + assert isinstance(got, tuple) and len(got) == 2, k + for j, t in enumerate((cos, sin)): + out[f"{k}.{j}"] = got[j] + ref = pool.per_rank(t.view(sp, 1, ROWS, rope_dim), ttnn.bfloat16, ttnn.TILE_LAYOUT, device=False) + expect(f"{k}.{j}", ref, ttnn.bfloat16, ttnn.TILE_LAYOUT, (1, 1, ROWS, rope_dim)) + for k, v in visible.items(): + ref = pool.per_rank(v.view(sp, 1, ROWS, 1), ttnn.float32, ttnn.TILE_LAYOUT, device=False) + expect(k, ref, ttnn.float32, ttnn.TILE_LAYOUT, (1, 1, ROWS, 1)) + # the embedding's token shape + token = ttnn.reshape(out["token"], (1, 1, ROWS)) + assert tuple(token.shape) == (1, 1, ROWS) and torch.equal( + _chips(token)[tp].view(-1).to(torch.int64), ints["token"][1].to(torch.int64) + ) + + +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(), indirect=True) +def test_v41_copy_batch(mesh_device, device_params): + """``ttnn.copy_host_to_device_tensors`` (the step's copies in one dispatch round) writes + every tensor of the batch as ``copy_host_to_device_tensor`` does, bit for bit on every chip, for the + step's three + shapes (the one-page packed row per rank replicated over TP, two Engram lookups of one real shard among + zero + shards); a second batch of other values replaces the first's; and after a 50 ms host sleep (the workers + asleep as + after the step's device wait) the batch's wall against the three single copies' (printed, not + asserted).""" + import time + + pool = _Pool(mesh_device) + sp, tp = pool.sp, pool.tp + chips = sp * tp + g = torch.Generator().manual_seed(128) + + def one_row(seed): + host = torch.randint(0, 2**31 - 1, (sp, 1, 1, 12800), generator=g, dtype=torch.int32) + seed + parts = [] + for r in range(sp): + shard = ttnn.Tensor(host[r : r + 1], ttnn.uint32) + parts.extend([shard] * tp) + return ttnn.from_host_shards(parts, ttnn.MeshShape(sp, tp)) + + zero = ttnn.from_torch(torch.zeros(1, 1, 192, PACKED_WIDTH, dtype=torch.uint16), dtype=ttnn.uint16) + + def engram(seed, chip): + rows = torch.randint(0, 60000, (1, 1, 192, PACKED_WIDTH), generator=g, dtype=torch.int32) + seed + parts = [zero] * chips + parts[chip] = ttnn.Tensor(rows.to(torch.uint16), ttnn.uint16) + return ttnn.from_host_shards(parts, ttnn.MeshShape(sp, tp)) + + def batch(seed): + return [one_row(seed), engram(seed, 5), engram(seed, 17)] + + first = batch(0) + devs = [ttnn.to_device(h, mesh_device) for h in first] + second = batch(1000) + ttnn.copy_host_to_device_tensors(second, devs) + for i, (h, d) in enumerate(zip(second, devs)): + assert d.dtype == h.dtype and d.layout == h.layout and tuple(d.shape) == tuple(h.shape), i + for chip, (mine, theirs) in enumerate(zip(_chips(d), _chips(h))): + assert torch.equal(mine, theirs), (i, chip) + # the single path on a third batch into the same buffers: the same bits as the batch of the same values + third = batch(2000) + ref = [ttnn.to_device(h, mesh_device) for h in third] + ttnn.copy_host_to_device_tensors(third, devs) + for i, (d, r) in enumerate(zip(devs, ref)): + for chip, (mine, theirs) in enumerate(zip(_chips(d), _chips(r))): + assert torch.equal(mine, theirs), (i, chip) + # the timing: after a host sleep (the workers blocked on their futex) the batch against three single + # copies + singles, batches = [], [] + for trial in range(10): + time.sleep(0.05) + t0 = time.perf_counter() + for h, d in zip(second, devs): + ttnn.copy_host_to_device_tensor(h, d) + singles.append(1e3 * (time.perf_counter() - t0)) + time.sleep(0.05) + t0 = time.perf_counter() + ttnn.copy_host_to_device_tensors(third, devs) + batches.append(1e3 * (time.perf_counter() - t0)) + ttnn.synchronize_device(mesh_device) + print(f"copy_batch: three single copies after a sleep {sorted(singles)} ms; one batch {sorted(batches)} ms") + + +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(), indirect=True) +def test_v41_engram_live_rows(mesh_device, device_params): + """``TtV41Engram._packed_shards`` with ``live_rows`` against the full path on a small random table.""" + sp, tp = tuple(mesh_device.shape) + chips = sp * tp + g = torch.Generator().manual_seed(59) + rows, head_dim = 4096, C.ENGRAM_HEAD_DIM + weight = torch.randn(rows, head_dim, generator=g).to(torch.float8_e4m3fn) + scale = (torch.rand(rows, head_dim // 32, generator=g) * 4).to(torch.float8_e8m0fnu) + e = TtV41Engram.__new__(TtV41Engram) + e.table, e.chips, e.sp, e.tp, e._zero_shards = V41EngramTable(weight, scale), chips, sp, tp, {} + cols = (C.ENGRAM_MAX_NGRAM_SIZE - 1) * C.ENGRAM_N_HEADS + chunk = ROWS * sp + lookups = chunk * cols + per_chip = lookups // chips + rows_per_chip = per_chip // cols + hash_ids = torch.randint(0, rows, (chunk, cols), generator=g) + live = torch.tensor(sorted({0, 3, ROWS + 1, 2 * ROWS, chunk - 1})) + full, _ = e._packed_shards(hash_ids, lookups) + part, keep = e._packed_shards(hash_ids, lookups, live_rows=live) + assert keep.shape == (len({int(r) // rows_per_chip for r in live}) * per_chip, PACKED_WIDTH) + live_chips = {int(r) // rows_per_chip for r in live} + for c, (a, b) in enumerate(zip(_chips(full), _chips(part))): + assert a.shape == b.shape == (1, 1, per_chip, PACKED_WIDTH), (c, a.shape, b.shape) + if c in live_chips: + assert torch.equal(a, b), c + else: + assert not b.any(), c diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_norm.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_norm.py new file mode 100644 index 000000000000..e89eb8fe26e9 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_decode_norm.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The decode sublayer norm on a core grid (``tt/v41/decode_norm.py``, ``V41_DECODE_NORM=sharded``) against +the +block's ``TtDistributedRmsNorm`` on the decode tile. + +No weights: a random gamma, ``V41_DN_TRIALS`` random ``[1, 1, 32, hidden/tp]`` bf16 tiles a chip (every chip's +TP +slice of its own rows), the two outputs compared on every chip (the interleaved op's rounding against the +sharded +program's: bf16-level), then each path under trace (``V41_DN_REPLAYS`` replays of 4 calls) for the per-site +device +time, for every core count of ``V41_DN_CORES`` (comma list; default 10,20,40). Results: ``V41_DN_RESULT``, +``V41_DN_OUT``.""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.tt_ccl import per_axis_topology +from models.demos.deepseek_v3_d_p.tt.tt_distributed_rms_norm import TtDistributedRmsNorm +from models.demos.deepseek_v3_d_p.tt.v41.ccl import V41Collectives, fabric_num_links +from models.demos.deepseek_v3_d_p.tt.v41.decode_norm import DecodeRmsNorm +from models.demos.deepseek_v3_d_p.tt.v41.layout import TP_AXIS + + +def _pcc(a: torch.Tensor, b: torch.Tensor) -> float: + a, b = a.flatten().double(), b.flatten().double() + return float(torch.corrcoef(torch.stack([a, b]))[0, 1]) + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=64 * 1024 * 1024), indirect=True) +def test_v41_decode_norm(mesh_device, device_params): + trials = int(os.environ.get("V41_DN_TRIALS", 4)) + replays = int(os.environ.get("V41_DN_REPLAYS", 20)) + core_counts = [int(c) for c in os.environ.get("V41_DN_CORES", "10,20,40").split(",")] + sp, tp = tuple(mesh_device.shape) + hidden = C.EMB_SIZE + width = hidden // tp + torch.manual_seed(5) + gamma = torch.rand(hidden) * 1.5 + 0.25 + norm = TtDistributedRmsNorm( + mesh_device=mesh_device, + emb_dim=hidden, + torch_weight=gamma, + epsilon=C.RMS_NORM_EPS, + cluster_axis=TP_AXIS, + num_links=fabric_num_links(), + topology=per_axis_topology()[TP_AXIS], + ) + ccl = V41Collectives(mesh_device) + decode_norms = {cores: DecodeRmsNorm(norm, width, ccl.tp_all_gather, cores) for cores in core_counts} + + def tile(h: torch.Tensor): + return ttnn.from_torch( + h.view(1, 1, sp * 32, hidden), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(2, 3)), + ) + + result = {"mesh": [sp, tp], "width": width, "trials": [], "cores": core_counts} + worst = {str(c): {"pcc": 1.0, "max_abs": 0.0, "rel": 0.0} for c in core_counts} + for trial in range(trials): + h = torch.randn(sp, 32, hidden) * (1.0 + trial) + x = tile(h) + ref = norm(x) + ref_t = ttnn.to_torch(ref, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, (sp, tp), dims=(2, 3))).float() + for cores, dn in decode_norms.items(): + out = dn(x) + assert tuple(out.shape) == tuple(ref.shape) and out.dtype == ref.dtype, (out.shape, out.dtype) + assert not out.memory_config().is_sharded() + out_t = ttnn.to_torch( + out, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, (sp, tp), dims=(2, 3)) + ).float() + max_abs = float((ref_t - out_t).abs().max()) + rel = max_abs / max(float(ref_t.abs().max()), 1e-6) + pcc = _pcc(ref_t, out_t) + entry = { + "trial": trial, + "cores": cores, + "pcc": round(pcc, 7), + "max_abs": round(max_abs, 5), + "rel": round(rel, 6), + } + result["trials"].append(entry) + logger.info(f"decode norm trial {json.dumps(entry)}") + w = worst[str(cores)] + w["pcc"], w["max_abs"], w["rel"] = min(w["pcc"], pcc), max(w["max_abs"], max_abs), max(w["rel"], rel) + ttnn.deallocate(out) + ttnn.deallocate(ref) + ttnn.deallocate(x) + result["worst"] = worst + # the per-site device time of each path, under trace + x = tile(torch.randn(sp, 32, hidden)) + timing = {} + paths = [("interleaved", lambda: norm(x))] + [ + (f"sharded-{c}", (lambda d=dn: d(x))) for c, dn in decode_norms.items() + ] + for name, fn in paths: + for _ in range(2): + ttnn.deallocate(fn()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn() for _ in range(4)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / 4 + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + timing[name] = round(best * 1e6, 1) + logger.info(f"decode norm traced {name}: {timing[name]} us a call (min of {replays} replays of 4 calls)") + result["site_us"] = timing + logger.info(f"V41_DN_RESULT {json.dumps(result)}") + out_path = os.environ.get("V41_DN_OUT") + if out_path: + with open(os.path.expanduser(out_path), "w") as f: + json.dump(result, f, indent=1) + for c, w in worst.items(): + assert w["pcc"] > 0.9999 and w["rel"] < 0.02, (c, w) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_defer_final.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_defer_final.py new file mode 100644 index 000000000000..322690fbe00f --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_defer_final.py @@ -0,0 +1,428 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Deferred final chunks through ``DeepseekV41ForCausalLM`` as the plugin drives it +(``TT_V41_DEFER_FINAL=1``): a cohort of prompts of mixed length run (a) serially as today (every chunk of +a prompt, then the next prompt; the joins at the first decode step) and (b) bulk / park / restore / final / +decode +(every prompt's chunks but the last, in admission order, which parks each one when the next takes the state; +then +the finals in the same order, each restoring its prefix from its slot; then decode), in one process, twice (a +second +cohort of other prompts reuses the slots of the first through the generator's LRU). Compared per prompt +between (a) +and (b): the first token (the argmax of the final chunk's logits), ``V41_DF_STEPS`` (8) greedy decode tokens, +and the +slot's pool rows after the decode steps (the compressed KV and index-K rows of every source, every layer's +ring in a +canonical frame, the candidate source's block pages, on the slot's rank's first chip), split at the rows the +decode +wrote: the prefill-written rows (the restored prefix, the final chunk, the carries) are the gate with the +first +token; the decode-written rows and the decode tokens are reported (two decodes of the same tokens at +different global steps differ in those rows' low bits, and a greedy token flips now and then; +``V41_DF_CONTROL=1`` +runs the serial order twice on the first cohort to show that variation on its own). Timed per final: the +restore +(``last_restore``), the final call's wall, the first decode step's wall (it holds the finals' joins) against +the +later steps' median. ``V41_DF_PROMPTS`` (``1028,8196,12292,40964,131076,131076``: 1, 2, 3, 8, 26 and 26 +chunks), +``V41_DF_LAYERS`` (``all``), ``V41_DF_BATCH`` (8 slots: one a rank, so the second cohort reuses slots), +``V41_DF_MAX_SEQ`` (1,053,696, the served context). Results: ``V41_DF_OUT``.""" + +import json +import os +import statistics +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_depth import _dram_bytes, parse_layers +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_generator_vllm import _page_table, _trace_bytes +from models.demos.deepseek_v3_d_p.tt.v41.generator_vllm import DeepseekV41ForCausalLM +from models.demos.deepseek_v3_d_p.tt.v41.pool import BLOCK_ROWS, RING, V41DecodePool +from models.demos.deepseek_v3_d_p.tt.v41.weights import resolve_checkpoint +from models.demos.deepseek_v3_d_p.utils.kv_cache_utils import MlaKvCacheFormat + +FP8 = MlaKvCacheFormat.SCALED_FP8.storage_dtype # the packed KV rows' dtype (FP8_E4M3) + + +def _chip_rows(mesh_device, t, a: int, b: int, chip: int) -> torch.Tensor: + """Rows ``[a, b)`` of ``t`` on ``chip`` as host bytes / values (FP8 bytes through the composer).""" + shape = list(t.shape) + whole = a == 0 and b == shape[2] and shape[0] == 1 # the slice would be the tensor itself + if whole: + piece = t + elif len(shape) == 4 and shape[0] > 1: # [users, 1, rows, W]: the caller passes absolute rows of user 0's frame + u, a2 = divmod(a, shape[2]) + piece = ttnn.slice(t, [u, 0, a2, 0], [u + 1, 1, a2 + (b - a), shape[3]]) + else: + piece = ttnn.slice(t, [0, 0, a, 0], [1, 1, b, shape[3]]) + if t.dtype == FP8: + host = ttnn.to_torch(piece, mesh_composer=ttnn.ConcatMeshToTensor(mesh_device, dim=0))[chip] + else: + host = ttnn.to_torch(ttnn.get_device_tensors(piece)[chip]) + if not whole: + ttnn.deallocate(piece) + return host.reshape(-1, shape[3]).contiguous() + + +def _slot_rows(mesh_device, pool: V41DecodePool, slot: int, length: int, steps: int) -> tuple[dict, dict]: + """A prompt of ``length`` tokens plus ``steps`` decode tokens in pool slot ``slot`` on its rank's first + chip (the + bytes as stored), and per key the row the decode-written rows start at: the prefill's rows (the restored + prefix, + the final chunk, the carries) are the exactness gate, the decode's rows (the window rows and the + compressed rows + of the ``steps`` tokens) are compared apart (they differ in their low bits between two decodes of the + same tokens at different global steps, the serial path's own property).""" + cfg = pool.config + rank, u = divmod(slot, pool.users) + chip = rank * pool.tp + out, bounds = {}, {} + for src, t in pool.kv.items(): + base = u * pool.slots[src].rows + if src >= 0: + ratio = cfg.compress_ratio(src) + n = (length + steps) // ratio + first = base + pool.comp_row(src) + out[f"kv[{src}]"] = _chip_rows(mesh_device, t, first, first + n, chip) + bounds[f"kv[{src}]"] = length // ratio + for layer in pool.slots[src].layers: + first = base + pool.ring_row(layer) + # the ring in a canonical frame: position p sits at ring slot (step_join + p - start) % RING, and + # the + # two orders join at global steps 8 apart (every ring differed by the rotation), so the rows + # are rolled by -pool.step, the same for both orders' readouts after the same number of steps; the + # last ``steps`` rows of the frame are the decode's + rows = _chip_rows(mesh_device, t, first, first + RING, chip) + out[f"ring[{layer}]"] = torch.roll(rows, -(pool.step % RING), dims=0) + bounds[f"ring[{layer}]"] = RING - steps + for src, t in pool.index_k.items(): + ratio = cfg.compress_ratio(src) + n = (length + steps) // ratio + cap = pool.slots[src].cap + 32 + out[f"index_k[{src}]"] = _chip_rows(mesh_device, t, u * cap, u * cap + n, chip) + bounds[f"index_k[{src}]"] = length // ratio + for src, t in pool.index_k_blocks.items(): + ratio = cfg.compress_ratio(src) + n = -(-((length + steps) // ratio) // BLOCK_ROWS) + cap = pool.slots[src].cap + 32 + first = u * cap // BLOCK_ROWS + out[f"index_k_blocks[{src}]"] = _chip_rows(mesh_device, t, first, first + n, chip) + bounds[f"index_k_blocks[{src}]"] = (length // ratio) // BLOCK_ROWS # the block the boundary row is in: decode's + return out, bounds + + +def _compare(want: dict, got: dict, bounds: dict) -> tuple[list[str], list[str]]: + """(the prefill-written rows' mismatches, the decode-written rows' mismatches), one line per key.""" + prefill, decode = [], [] + for k, a in want.items(): + b = got[k] + if a.shape != b.shape: + prefill.append(f"{k}: shape {tuple(a.shape)} vs {tuple(b.shape)}") + continue + r = bounds[k] + for part, x, y, out in (("prefill", a[:r], b[:r], prefill), ("decode", a[r:], b[r:], decode)): + if x.numel() and not torch.equal(x, y): + d = x != y + rows = d.any(-1).nonzero().flatten() + out.append( + f"{k} {part} rows: {int(d.sum())} of {x.numel()} differ, rows {int(rows.min())}..{int(rows.max())}" + f" of {x.shape[0]}" + ) + return prefill, decode + + +def _trace_region() -> int: + return 2 * 1024**3 + + +@pytest.mark.timeout(int(os.environ.get("V41_DF_TIMEOUT", 2 * 3600))) +@pytest.mark.parametrize( + "mesh_device, device_params", + galaxy_meshes(trace_region_size=int(os.environ.get("V41_DF_TRACE_REGION", _trace_region()))), + indirect=True, +) +def test_v41_defer_final(mesh_device, device_params, monkeypatch): + layers = parse_layers(os.environ.get("V41_DF_LAYERS", "all")) + lengths = [int(x) for x in os.environ.get("V41_DF_PROMPTS", "1028,8196,12292,40964,131076,131076").split(",")] + steps = int(os.environ.get("V41_DF_STEPS", 8)) + batch = int(os.environ.get("V41_DF_BATCH", 8)) + chunk = int(os.environ.get("TT_V41_CHUNK", 5120)) + max_seq_len = int(os.environ.get("V41_DF_MAX_SEQ", 1053696)) + monkeypatch.setenv("TT_V41_CHUNK", str(chunk)) + monkeypatch.setenv("TT_V41_LAYERS", ",".join(str(l) for l in layers)) + monkeypatch.setenv("TT_V41_DEFER_FINAL", "1") + monkeypatch.setenv("TT_V41_PREFILL_GENERIC", "1") + monkeypatch.setenv("TT_V41_PREFILL_SLOTS", os.environ.get("TT_V41_PREFILL_SLOTS", "1")) + monkeypatch.setenv("TT_V41_SHORT_CHUNK", os.environ.get("TT_V41_SHORT_CHUNK", "0")) + traced = os.environ.get("V41_DF_TRACE", "1") == "1" # 0: no capture at all + if "TT_V41_TRACE_PREFILL_LENGTHS" not in os.environ: + monkeypatch.setenv("TT_V41_TRACE_PREFILL_LENGTHS", "1028,8196,131076" if traced else "") + monkeypatch.setenv("TT_V41_TRACE_PREFILL_SLACK", "0") + if resolve_checkpoint() is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + sp = tuple(mesh_device.shape)[0] + assert batch % sp == 0 and len(lengths) <= batch, (batch, sp, lengths) + result = { + "layers": layers, + "lengths": lengths, + "steps": steps, + "batch": batch, + "chunk": chunk, + "max_seq": max_seq_len, + "decode_flags": {k: v for k, v in os.environ.items() if k.startswith("V41_DECODE_")}, + "cohorts": [], + } + + t0 = time.time() + model = DeepseekV41ForCausalLM.initialize_vllm_model( + None, mesh_device, max_batch_size=batch, max_seq_len=max_seq_len + ) + result["build_s"] = round(time.time() - t0, 1) + assert model.decoder.pool.parked, "TT_V41_DEFER_FINAL=1: the parking buffers" + kv = model.allocate_kv_cache((16, 1, 64, 512), torch.bfloat16, len(layers)) + t0 = time.time() + model.warmup_model_prefill(enable_trace=False, kv_cache=kv, can_sample_on_device=False) + model.warmup_model_decode( + enable_trace=False, kv_cache=kv, max_batch_size=batch, num_blocks=16, can_sample_on_device=False + ) + if traced: + model.warmup_model_prefill(enable_trace=True, kv_cache=kv, can_sample_on_device=False) + model.warmup_model_decode( + enable_trace=True, kv_cache=kv, max_batch_size=batch, num_blocks=16, can_sample_on_device=False + ) + result["traced"] = traced + result["warmup_s"] = round(time.time() - t0, 1) + result["dram_after_warmup"] = _dram_bytes(mesh_device) + result["trace_after_warmup"] = _trace_bytes(mesh_device) + result["prefill_traces"] = len(model.prefill_traces) + logger.info( + f"defer-final: built in {result['build_s']} s, warm-up {result['warmup_s']} s;" + f" DRAM {result['dram_after_warmup'] / 2**30:.2f} GiB" + ) + + need = 2 * (sum(lengths) + len(lengths) * (steps + 64)) + try: + text = orc.text_tokens(need)[0] + except ValueError as e: + have = int(str(e).split()[0]) + book = orc.text_tokens(have)[0] + text = book.repeat(-(-need // have))[:need] + + sample_on_device = DeepseekV41ForCausalLM.SAMPLE_ON_DEVICE + + class _Params: + def __init__(self, n): + self.temperature, self.top_k, self.top_p = [0.0] * n, [1] * n, [0.0] * n + self.repetition_penalty = 1.0 + + def prefill_call(block: int, tokens: torch.Tensor, start: int, end: int) -> tuple[torch.Tensor, float]: + t0 = time.time() + out = model.prefill_forward( + tokens=tokens.view(1, -1).to(torch.int32), + prompt_lens=torch.tensor([end]), + page_table=_page_table([block]), + start_pos=torch.tensor([start], dtype=torch.int32), + kv_cache=kv, + enable_trace=True, + prompt_total_lens=torch.tensor([int(tokens.numel())]), + ) + return out[0, 0], time.time() - t0 + + def decode(rows: list[tuple[int, int, int]]) -> tuple[list[int], float]: + t0 = time.time() + out = model.decode_forward( + tokens=torch.tensor([[t] for _, t, _ in rows], dtype=torch.int32), + start_pos=torch.tensor([p for _, _, p in rows], dtype=torch.int32), + page_table=_page_table([b for b, _, _ in rows]), + kv_cache=kv, + enable_trace=True, + read_from_device=True, + sampling_params=_Params(len(rows)) if sample_on_device else None, + ) + wall = time.time() - t0 + if sample_on_device: + return [int(x) for x in out], wall + return [int(out[i, 0].argmax()) for i in range(len(rows))], wall + + def run_order(order: str, prompts: list[tuple[int, torch.Tensor]]) -> dict: + """``prompts``: (block, tokens). Returns per block the first token, the decode tokens, the slot's + rows.""" + rec = {"order": order, "finals": {}, "steps": [], "slots": {}, "bounds": {}} + first: dict[int, int] = {} + t_all = time.time() + if order.startswith("serial"): + for block, tokens in prompts: + n = int(tokens.numel()) + for start in range(0, n, chunk): + logits, wall = prefill_call(block, tokens, start, min(start + chunk, n)) + if start + chunk >= n: + first[block] = int(logits.argmax()) + rec["finals"][block] = {"wall_s": round(wall, 3), "mode": model.prefill_last["mode"]} + else: + for block, tokens in prompts: # the bulk chunks, in admission order + n = int(tokens.numel()) + prefix = (n - 1) // chunk * chunk + for start in range(0, prefix, chunk): + prefill_call(block, tokens, start, start + chunk) + t_finals = time.time() + for block, tokens in prompts: # the finals, FIFO + n = int(tokens.numel()) + prefix = (n - 1) // chunk * chunk + logits, wall = prefill_call(block, tokens, prefix, n) + first[block] = int(logits.argmax()) + rec["finals"][block] = { + "wall_s": round(wall, 3), + "mode": model.prefill_last["mode"], + "restore": dict(model.decoder.last_restore) if prefix > 0 else None, + } + model.decoder.last_restore = {} + # the finals' phase: every restore, final chunk and join but the last request's (at the first + # step) + rec["finals_phase_s"] = round(time.time() - t_finals, 3) + rec["prefill_phase_s"] = round(time.time() - t_all, 2) + fed = dict(first) + tokens_out: dict[int, list[int]] = {b: [] for b, _ in prompts} + for s in range(steps): + rows = [(b, fed[b], int(t.numel()) + s) for b, t in prompts] + ids, wall = decode(rows) + rec["steps"].append(round(wall, 4)) + for (b, _, _), tok in zip(rows, ids, strict=True): + tokens_out[b].append(tok) + fed[b] = tok + rec["first"] = first + rec["tokens"] = tokens_out + rec["slot_of"] = {b: model.slots[b] for b, _ in prompts} + for b, t in prompts: + rec["slots"][b], rec["bounds"][b] = _slot_rows( + mesh_device, model.decoder.pool, model.slots[b], int(t.numel()), steps + ) + return rec + + def compare(cohort: int, name: str, want: dict, got: dict, prompts, control: dict | None) -> dict: + """``got`` (an order) against ``want`` (the serial order): per block the first token, the decode + tokens, + the prefill-written slot rows and the decode-written rows. With ``control`` (the serial order's own + repeat + compared the same way) this is the gate: the first token equal, and every prefill-row mismatch one the + serial path shows against itself (an eager final chunk of 2,052 tokens leaves its last ~770 + rows and the rings different between two serial runs of the same prompt).""" + out = {} + for b, t in prompts: + prefill_bad, decode_bad = _compare(want["slots"][b], got["slots"][b], got["bounds"][b]) + cmp = { + "first_equal": want["first"][b] == got["first"][b], + "tokens_equal": want["tokens"][b] == got["tokens"][b], + "slot_mismatches": prefill_bad, + "decode_row_mismatches": decode_bad, + "text_serial": model.tokenizer.decode([want["first"][b]] + want["tokens"][b]), + "text_got": model.tokenizer.decode([got["first"][b]] + got["tokens"][b]), + } + if control is not None: + # a prompt the serial path does not reproduce itself (the eager 2,052-token final chunk's + # last rows, and which rings, vary run to run) is reported, not gated; the others must be + # exact + cmp["control_clean"] = not control[str(b)]["slot_mismatches"] + cmp["beyond_control"] = prefill_bad if cmp["control_clean"] else [] + out[str(b)] = cmp + if control is not None and not (cmp["first_equal"] and not cmp["beyond_control"]): + failures.append(f"cohort {cohort} {name} block {b} ({int(t.numel())} tokens): {cmp}") + logger.info( + f"defer-final cohort {cohort} {name} block {b} ({int(t.numel())} tokens): first equal" + f" {cmp['first_equal']}, tokens equal {cmp['tokens_equal']}, prefill-row mismatches" + f" {len(cmp['slot_mismatches'])}, decode-row mismatches {len(cmp['decode_row_mismatches'])}:" + f" {cmp['text_got']!r}" + ) + if cmp["slot_mismatches"]: + logger.info(f"defer-final cohort {cohort} {name} block {b} prefill rows: {cmp['slot_mismatches']}") + if cmp["decode_row_mismatches"]: + logger.info(f"defer-final cohort {cohort} {name} block {b} decode rows: {cmp['decode_row_mismatches']}") + if not cmp["tokens_equal"]: + logger.info( + f"defer-final cohort {cohort} {name} block {b} tokens: serial {want['first'][b]} {want['tokens'][b]}" + f" / {name} {got['first'][b]} {got['tokens'][b]}" + ) + return out + + def summary(r: dict) -> dict: + return { + "finals": {str(b): v for b, v in r["finals"].items()}, + "steps_s": r["steps"], + "step_median_s": round(statistics.median(r["steps"][1:]) if len(r["steps"]) > 1 else r["steps"][0], 4), + "first_step_s": r["steps"][0], + "prefill_phase_s": r["prefill_phase_s"], + "finals_phase_s": r.get("finals_phase_s"), + "slot_of": {str(b): s for b, s in r["slot_of"].items()}, + "first": {str(b): t for b, t in r["first"].items()}, + "tokens": {str(b): t for b, t in r["tokens"].items()}, + } + + failures = [] + offset = 0 + control = os.environ.get("V41_DF_CONTROL", "1") == "1" # the serial order twice per cohort: its own variation + slots_probe = os.environ.get("V41_DF_SLOTS", "0") == "1" # the serial order in other slots + for cohort in range(2): + prompts = [] + for i, n in enumerate(lengths): + block = 100 * (cohort + 1) + i + 1 + prompts.append((block, text[offset : offset + n])) + offset += n + steps + 64 + serial = run_order("serial", prompts) + rec = {"cohort": cohort, "blocks": [b for b, _ in prompts], "lengths": lengths, "serial": summary(serial)} + own = None + if control: + serial2 = run_order("serial2", prompts) + rec["serial2"] = summary(serial2) + own = rec["compare_serial2"] = compare(cohort, "serial2", serial, serial2, prompts, None) + del serial2 + if slots_probe and cohort == 0: + # V41_DF_SLOTS=1 (the served serial path gave other outputs for the same prompts after a + # different first point, i.e. in other slots): the serial order on the same prompts under new keys + # pinned to the slots of the next prompt (another rank, another previous occupant), then under new + # keys pinned to each prompt's original slot (the same rank, the previous occupant now the prompt + # before it). serial3 != serial says the output depends on the slot; serial4 == serial then says + # it + # is the rank's arithmetic, serial4 != serial that the slot's previous occupant leaks into it. + n = len(prompts) + for name, shift, base in (("serial3", 1, 1000), ("serial4", 0, 2000)): + pinned = [(base + b, t) for b, t in prompts] + for i, (nb, _) in enumerate(pinned): + model.slots[nb] = serial["slot_of"][prompts[(i + shift) % n][0]] + r = run_order(name, pinned) + r = { + k: ( + {(b - base): v for b, v in val.items()} + if k in ("finals", "slots", "bounds", "slot_of", "first", "tokens") + else val + ) + for k, val in r.items() + } + rec[name] = summary(r) + rec[f"compare_{name}"] = compare(cohort, name, serial, r, prompts, None) + del r + deferred = run_order("defer", prompts) + rec["defer"] = summary(deferred) + rec["compare"] = compare(cohort, "defer", serial, deferred, prompts, own) + result["cohorts"].append(rec) + del serial, deferred + + result["failures"] = failures + result["dram_end"] = _dram_bytes(mesh_device) + out = os.environ.get("V41_DF_OUT") + if out: + with open(out, "w") as f: + json.dump(result, f, indent=1) + logger.info("defer-final result: " + json.dumps({k: v for k, v in result.items() if k != "cohorts"})) + for trace in model.prefill_traces.values(): + trace.release() + model.decoder.release() + assert not failures, failures diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_depth.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_depth.py new file mode 100644 index 000000000000..ad35c61b5990 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_depth.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""V4.1 prefill at an arbitrary layer depth on real weights: fit, time and next-token sanity (harness). + +The production gate runs the six-layer schedule the CPU oracle can afford. This test takes any layer schedule +(``V41_DEPTH_LAYERS``: a comma list with ranges, ``all`` = every backbone layer; Engram layers 1 and 14 get +the +checkpoint tables through ``engram_checkpoint``), prefills ``V41_DEPTH_SEQ`` tokens of real text in chunks of +``V41_DEPTH_CHUNK`` on the Galaxy, and reports: + +* bit-identical repeat (gated, like every V4.1 gate); +* wall time per untraced prefill, tokens -> last-N logits on the host, first run (kernel compile) and repeats; +* trace mode (``V41PrefillTrace``, one scored row as in generation; ``V41_DEPTH_TRACE=0`` skips it): wall time + per replayed prefill and device-only replay time, the bit-identical check against the untraced logits; +* next-token top-1 / top-5 accuracy and mean log-probability of the real text over the last +``V41_DEPTH_SCORED`` + positions, against the prompt's own next tokens: no reference needed, so it runs where the oracle cannot (a + working model predicts English far above chance; a broken one does not). Reported, not gated; +* per-chip DRAM in use after the build and after the prefill, and host RSS. + +Results go to the log as ``V41_DEPTH_RESULT {json}`` and to ``V41_DEPTH_OUT`` when set. +""" + +import json +import os +import resource +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.engram_checkpoint import build_engram_modules +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer, V41PrefillTrace +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint + +DEFAULT_LAYERS = "0,2,3,20,21,24" + + +def parse_layers(spec: str) -> list[int]: + if spec.strip() == "all": + return list(range(C.NUM_LAYERS)) + out: set[int] = set() + for part in spec.split(","): + part = part.strip() + if "-" in part: + a, b = part.split("-") + out |= set(range(int(a), int(b) + 1)) + elif part: + out.add(int(part)) + return sorted(out) + + +def _dram_bytes(mesh_device) -> int | None: + try: # as tests/v41/test_v41_expert_dtype.py: per chip (the view is one chip of the mesh) + view = ttnn.get_memory_view(mesh_device, ttnn.BufferType.DRAM) + return int(view.total_bytes_allocated_per_bank * view.num_banks) + except Exception as e: # the API moves; the number is a report, not a gate + logger.warning(f"DRAM statistics unavailable: {e}") + return None + + +def _rss_gb() -> float: + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1e6 + + +@pytest.mark.timeout(int(os.environ.get("V41_DEPTH_TIMEOUT", 6 * 3600))) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(), indirect=True) +def test_v41_depth(mesh_device, device_params): + layers = parse_layers(os.environ.get("V41_DEPTH_LAYERS", DEFAULT_LAYERS)) + seq = int(os.environ.get("V41_DEPTH_SEQ", 8192)) + chunk = int(os.environ.get("V41_DEPTH_CHUNK", min(seq, 8192))) + scored = int(os.environ.get("V41_DEPTH_SCORED", 256)) + repeats = int(os.environ.get("V41_DEPTH_REPEATS", 3)) + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + from transformers import AutoTokenizer + + result = { + "layers": layers, + "n_layers": len(layers), + "seq": seq, + "chunk": chunk, + "scored": scored, + "mesh": list(mesh_device.shape), + "fabric": str(device_params.get("fabric_config")), + } + spec = orc.real_spec(tuple(layers), seq, checkpoint=ckpt.root) # identity for the weight cache only + root = weight_cache_dir(spec, mesh_device.shape) + tokens = orc.text_tokens(seq)[0] + tokenizer = AutoTokenizer.from_pretrained(orc.HF_SNAPSHOT) + + t0 = time.time() + engram, engram_hash = build_engram_modules(mesh_device, C, ckpt, layers, tokenizer) + result["engram_layers"] = sorted(engram) + result["engram_build_s"] = round(time.time() - t0, 1) + result["rss_after_engram_gb"] = round(_rss_gb(), 1) + + top = ckpt.read(["embed.weight", "norm.weight", "head.weight"]) + t0 = time.time() + model = TtV41Transformer( + mesh_device, + C, + layers, + lambda layer, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, layer), + top["embed.weight"], + top["norm.weight"], + top["head.weight"], + max_seq_len=-(-seq // chunk) * chunk, + chunk=chunk, + engram=engram, + engram_hash=engram_hash, + weight_cache_path=root, + ) + result["build_s"] = round(time.time() - t0, 1) + result["dram_after_build_bytes"] = _dram_bytes(mesh_device) + result["rss_after_build_gb"] = round(_rss_gb(), 1) + logger.info(f"depth: built {len(layers)} layers in {result['build_s']} s; DRAM {result['dram_after_build_bytes']}") + + times = [] + first = None + for i in range(repeats): + t0 = time.time() + logits, state = model.prefill(tokens, logit_positions=scored) + times.append(round(time.time() - t0, 3)) + logger.info(f"depth: prefill {i} of {seq} tokens, {len(layers)} layers: {times[-1]} s") + if first is None: + first = logits.clone() + else: + assert torch.equal(first, logits), f"prefill {i} differs from the first (not bit-identical)" + del state + result["prefill_s"] = times + result["prefill_first_s"] = times[0] + result["prefill_repeat_s"] = min(times[1:]) if len(times) > 1 else None + result["dram_after_prefill_bytes"] = _dram_bytes(mesh_device) + + if os.environ.get("V41_DEPTH_TRACE", "1") == "1": + t0 = time.time() + trace = V41PrefillTrace(model, tokens, 1) + result["trace_capture_s"] = round(time.time() - t0, 1) + result["trace_segments"] = trace.num_segments + traced_times = [] + for i in range(repeats): + t0 = time.time() + traced_logits, _ = trace.run(tokens) + traced_times.append(round(time.time() - t0, 3)) + # the traced last-row logits equal the untraced ones (the trace test checks every cache; this checks + # what + # generation would consume) + assert torch.equal(traced_logits, first[-1:]), "traced last-row logits differ from the untraced prefill" + device_times = [] + for i in range(repeats): + ttnn.synchronize_device(mesh_device) + t0 = time.time() + for controller in trace.controllers: + controller.replay(blocking=False) + ttnn.synchronize_device(mesh_device) + device_times.append(round(time.time() - t0, 3)) + trace.release() + result["traced_prefill_s"] = traced_times + result["traced_prefill_min_s"] = min(traced_times) + result["traced_device_only_s"] = device_times + result["traced_device_only_min_s"] = min(device_times) + logger.info(f"depth: traced prefill {traced_times} s, device-only replay {device_times} s") + + # positions seq-scored .. seq-1 predict tokens seq-scored+1 .. seq (the last one has no target) + logp = torch.log_softmax(first.float(), dim=-1)[:-1] + targets = tokens[seq - scored + 1 :] + top5 = logp.topk(5, dim=-1).indices + result["top1_acc"] = round((top5[:, 0] == targets).float().mean().item(), 4) + result["top5_acc"] = round((top5 == targets[:, None]).any(-1).float().mean().item(), 4) + result["mean_logprob"] = round(logp.gather(1, targets[:, None]).mean().item(), 4) + result["ppl"] = round(float(torch.exp(-logp.gather(1, targets[:, None]).mean())), 2) + result["argmax_tokens"] = tokenizer.decode(top5[:64, 0].tolist()) + result["target_tokens"] = tokenizer.decode(targets[:64].tolist()) + result["rss_peak_gb"] = round(_rss_gb(), 1) + logger.info("V41_DEPTH_RESULT " + json.dumps(result)) + out = os.environ.get("V41_DEPTH_OUT") + if out: + with open(out, "w") as f: + json.dump(result, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_dram_probe.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_dram_probe.py new file mode 100644 index 000000000000..6ede018acae3 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_dram_probe.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The allocator's view of a chip's DRAM at the served trace region: the capacity the +allocator manages (per bank x banks), what is allocated and free right after the mesh opens, and the trace +region. +No weights; 20 s. ``V41_DRAM_TRACE_REGION`` (bytes, default 2 GiB, the served catalog's).""" + +import json +import os + +import pytest +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes + +TRACE_REGION = int(os.environ.get("V41_DRAM_TRACE_REGION", 2 * 1024**3)) + + +def _view(mesh_device, kind): + v = ttnn.get_memory_view(mesh_device, kind) + out = {} + for attr in dir(v): + if attr.startswith("_") or attr in ("blocks", "block_table"): + continue + try: + val = getattr(v, attr) + if isinstance(val, (int, float)): + out[attr] = int(val) + except Exception: + pass + out["fields"] = [a for a in dir(v) if not a.startswith("_")] + return out + + +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=TRACE_REGION), indirect=True) +def test_v41_dram_probe(mesh_device, device_params): + out = {"trace_region_bytes": TRACE_REGION} + for attr in ("dram_size_per_channel", "num_dram_channels", "dram_grid_size", "l1_size_per_core"): + try: + v = getattr(mesh_device, attr)() + out[attr] = int(v) if not hasattr(v, "x") else [int(v.x), int(v.y)] + except Exception as e: # not every binding exposes it + out[attr] = str(e) + for name, kind in (("dram", ttnn.BufferType.DRAM), ("trace", ttnn.BufferType.TRACE), ("l1", ttnn.BufferType.L1)): + try: + out[name] = _view(mesh_device, kind) + except Exception as e: # the API moves + out[name] = {"error": str(e)} + for name in ("dram", "trace"): + v = out[name] + if "num_banks" in v: + for k in list(v): + if k.endswith("_per_bank"): + out[f"{name}_{k[:-9]}_bytes"] = v[k] * v["num_banks"] + if isinstance(out.get("dram_size_per_channel"), int) and isinstance(out.get("num_dram_channels"), int): + out["dram_physical_bytes"] = out["dram_size_per_channel"] * out["num_dram_channels"] + logger.info(f"V41_DRAM_PROBE {json.dumps(out)}") + path = os.environ.get("V41_DRAM_OUT") + if path: + with open(os.path.expanduser(path), "w") as f: + json.dump(out, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_expert_dtype.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_expert_dtype.py new file mode 100644 index 000000000000..25d26b1558c6 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_expert_dtype.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Routed-expert device dtype study (dev-spec D-B, bead 8y7.9): bfp8 vs bfp4 on real layer 2 (KV + index source) +and real layer 0 (sliding-window only), same G1 bars for both dtypes. + +* MoE: ``TtV41Moe`` on one production 5120-token chunk of the layer's real MoE input (``expert_dtype_reference``: + the block oracle's ``ffn_in``, tiled); G1 bars routed >= 0.96, final >= 0.982 (plus gate / shared bars, which + do not depend on the expert dtype); bit-identical repeat; warm device time; DRAM allocated per chip by the MoE. +* Block: ``TtV41Block`` alone on the block oracle (2048 tokens, one chunk, teacher-forced ``x_in`` / ``pre_in``, + BF16 KV); G1 bar >= 0.98 real; bit-identical repeat; warm device time; DRAM per chip. + +Results are logged as ``V41_DTYPE_MOE`` / ``V41_DTYPE_BLOCK`` JSON lines. Device MoE weight caches are shared with +``test_block_v41`` (same cache root per schedule); their file names carry the dtype, and a dtype-specific marker +records a completed build. +""" + +import json +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tests.v41 import expert_dtype_reference as R +from models.demos.deepseek_v3_d_p.tests.v41.test_block_v41 import _pack, _pcc, _unpack +from models.demos.deepseek_v3_d_p.tests.v41.test_moe_v41 import BARS as MOE_BARS +from models.demos.deepseek_v3_d_p.tests.v41.test_moe_v41 import _metrics, _run +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.block import TtV41Block +from models.demos.deepseek_v3_d_p.tt.v41.cache import V41PrefillState +from models.demos.deepseek_v3_d_p.tt.v41.moe import TtV41Moe +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint +from models.demos.deepseek_v3_d_p.utils.fast_cache_checker import init_checker +from models.demos.deepseek_v3_d_p.utils.kv_cache_utils import MlaKvCacheFormat + +EXPERT_DTYPES = {"bfp8": ttnn.bfloat8_b, "bfp4": ttnn.bfloat4_b} +LAYERS = [2, 0] +BLOCK_BAR_REAL = 0.98 +TIMED_ITERS = 3 +MESH = [ + pytest.param( + (2, 4), + fabric2d_device_params(), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) +] +# test_block_v41's config at its real-weight oracle (candidate count matched to the oracle) +BLOCK_CONFIG = type("V41TestConfig", (C,), {"CANDIDATE_TOPK_BLOCKS": 96}) + + +def _dram_bytes_per_chip(mesh_device) -> int: + view = ttnn.get_memory_view(mesh_device, ttnn.BufferType.DRAM) + return view.total_bytes_allocated_per_bank * view.num_banks + + +def _weights(ckpt, layer: int, dtype: str, mesh_shape, spec=None): + """The device-weight cache root (tests/v41/weight_cache.py key) for the layer's schedule (or ``spec``) at the + ``dtype`` routed experts on ``mesh_shape``, the layer's weights (routed experts only when that cache is + incomplete) and the marker to touch after a completed build.""" + spec = spec or R.block_spec(layer) + root = weight_cache_dir(spec, mesh_shape, EXPERT_DTYPES[dtype]) + root.mkdir(parents=True, exist_ok=True) + init_checker(root) + marker = root / f"layer_{layer}.complete" + cached = marker.exists() + start = time.perf_counter() + w = load_layer_dense(ckpt, layer) if cached else load_layer(ckpt, layer) + logger.info(f"layer {layer} weights loaded (routed experts: {not cached}) {time.perf_counter() - start:.1f}s") + return root, w, marker + + +def _timed(fn, mesh_device, iters: int = TIMED_ITERS) -> float: + """Mean warm device time (ms) of ``fn`` with a device synchronize around the loop.""" + ttnn.synchronize_device(mesh_device) + start = time.perf_counter() + for _ in range(iters): + fn() + ttnn.synchronize_device(mesh_device) + return (time.perf_counter() - start) * 1e3 / iters + + +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("expert_dtype", list(EXPERT_DTYPES)) +@pytest.mark.parametrize("layer", LAYERS, ids=[f"L{l}" for l in LAYERS]) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_moe_expert_dtype(mesh_device, device_params, layer, expert_dtype): + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + ref = R.moe_parts(layer) + root, w, marker = _weights(ckpt, layer, expert_dtype, mesh_device.shape) + base = _dram_bytes_per_chip(mesh_device) + start = time.perf_counter() + moe = TtV41Moe( + mesh_device, + C, + layer, + w, + R.CHUNK, + routed_expert_weights_dtype=EXPERT_DTYPES[expert_dtype], + weight_cache_path=root, + ) + marker.touch() + del w + built = _dram_bytes_per_chip(mesh_device) + logger.info(f"MoE L{layer} {expert_dtype} built {time.perf_counter() - start:.1f}s") + + start = time.perf_counter() + first = _run(moe, mesh_device, ref["x"]) + repeat = _run(moe, mesh_device, ref["x"]) + logger.info(f"MoE L{layer} {expert_dtype} two forwards {time.perf_counter() - start:.1f}s") + for name in ("final", "routed", "shared", "logits", "weights", "indices"): + assert torch.isfinite(first[name].float()).all(), f"non-finite {name}" + assert torch.equal(first[name], repeat[name]), f"{name} differs between repeated runs" + + shape = tuple(mesh_device.shape) + tt_x = ttnn.from_torch( + ref["x"][None, None], + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=(2, 3)), + ) + ms = _timed(lambda: moe(tt_x), mesh_device) + report = { + "mesh": list(shape), + "layer": layer, + "block_type": C.block_type(layer).value, + "expert_dtype": expert_dtype, + "tokens": R.CHUNK, + "metrics": _metrics(ref, first) | {"ms": round(ms, 2)}, + "dram_gb_per_chip_moe": round((built - base) / 1e9, 3), + "dram_gb_per_chip_total": round(built / 1e9, 3), + } + logger.info(f"V41_DTYPE_MOE {json.dumps(report)}") + failed = {k: v for k, v in report["metrics"].items() if k in MOE_BARS and v < MOE_BARS[k]} + assert not failed, f"below G1 bars {MOE_BARS}: {failed}; {report}" + + +@pytest.mark.timeout(3600) +@pytest.mark.parametrize("expert_dtype", list(EXPERT_DTYPES)) +@pytest.mark.parametrize("layer", LAYERS, ids=[f"L{l}" for l in LAYERS]) +@pytest.mark.parametrize("mesh_device, device_params", MESH, indirect=True) +def test_v41_block_expert_dtype(mesh_device, device_params, layer, expert_dtype): + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + spec = R.block_spec(layer) + start = time.perf_counter() + result = orc.oracle(spec, orc.random_tokens(spec)) + rec = result["blocks"][layer] + logger.info(f"block oracle loaded {time.perf_counter() - start:.1f}s") + seq, cfg, shape = R.ORACLE_SEQ, BLOCK_CONFIG, tuple(mesh_device.shape) + tp, n = shape[1], cfg.HC_MULT + root, w, marker = _weights(ckpt, layer, expert_dtype, mesh_device.shape) + base = _dram_bytes_per_chip(mesh_device) + start = time.perf_counter() + block = TtV41Block( + mesh_device, + cfg, + layer, + w, + seq, + routed_expert_weights_dtype=EXPERT_DTYPES[expert_dtype], + weight_cache_path=root, + ) + marker.touch() + del w + built = _dram_bytes_per_chip(mesh_device) + logger.info(f"block L{layer} {expert_dtype} built {time.perf_counter() - start:.1f}s") + + def to_device(t, dims): + return ttnn.from_torch( + t, + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, shape, dims=dims), + ) + + x = to_device(_pack(rec["x_in"].float(), tp), (2, 3)) + pre = to_device(rec["pre_in"].float()[None, None], (2, None)) + down = lambda t: ttnn.to_torch(t, mesh_composer=ttnn.ConcatMesh2dToTensor(mesh_device, shape, dims=(2, 3))) + + def run(): + state = V41PrefillState(mesh_device, cfg, seq, seq, [layer], kv_format=MlaKvCacheFormat.BF16_RM) + x_out, pre_out = block(x, pre, state, seq) + return state, _unpack(down(x_out)[0, 0], n, tp), down(pre_out)[0, 0, :, :n] + + start = time.perf_counter() + _, x_out, pre_out = run() + _, x_rep, pre_rep = run() + logger.info(f"block L{layer} {expert_dtype} two forwards {time.perf_counter() - start:.1f}s") + assert torch.isfinite(x_out).all() and torch.isfinite(pre_out).all() + deterministic = torch.equal(x_out, x_rep) and torch.equal(pre_out, pre_rep) + state = V41PrefillState(mesh_device, cfg, seq, seq, [layer], kv_format=MlaKvCacheFormat.BF16_RM) + ms = _timed(lambda: block(x, pre, state, seq), mesh_device) + report = { + "mesh": list(shape), + "layer": layer, + "block_type": C.block_type(layer).value, + "expert_dtype": expert_dtype, + "tokens": seq, + "block_out": _pcc(rec["x_out"], x_out), + "pre_mix": _pcc(rec["pre_out"], pre_out), + "deterministic": deterministic, + "ms": round(ms, 2), + "dram_gb_per_chip_block": round((built - base) / 1e9, 3), + "dram_gb_per_chip_total": round(built / 1e9, 3), + } + logger.info(f"V41_DTYPE_BLOCK {json.dumps(report)}") + assert deterministic, report + assert report["block_out"] >= BLOCK_BAR_REAL, report diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_flash_sdpa.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_flash_sdpa.py new file mode 100644 index 000000000000..77db529a3d79 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_flash_sdpa.py @@ -0,0 +1,379 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The decode step's flash-decode SDPA (``tt/v41/flash_sdpa.py``, ``V41_DECODE_SDPA=flash``) +against ``sparse_sdpa`` on the chip's own heads. + +No weights, the decode tile: ``V41_SL_TRIALS`` random bf16 ``[1, 1, 32, 16 * 512]`` query tiles a chip (every +chip its +own), random RoPE tables, a random KV cache in both served formats (BF16 rows and the pool's packed scaled +FP8), +index rows in the c1 pattern (one live row a mesh row, the others sentinels), the ``k2`` / ``k3`` patterns +(the live +row and K draft rows of the same user, each with its own index row: the speculative verify step's layout) and +the +32-row pattern (every row live, varying counts). The two SDPAs' ``[1, 32, 32, 512]`` outputs are compared on +the live +rows (max abs difference, max relative to the largest output, PCC; the accumulation order differs, so bit +equality +is not expected), the dead rows of the flash output checked zero; then each SDPA alone under trace +(``V41_SL_REPLAYS`` +replays of ``V41_SL_CALLS`` calls) per pattern and format, and the whole local chain (``q_heads``, SDPA, +``o_heads``) +for the c1 pattern. Results: ``V41_FS_RESULT``, ``V41_SL_OUT``.""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.mla.rope import get_rot_transformation_mat +from models.demos.deepseek_v3_d_p.tt.v41.attention import SDPA_MIN_HEADS +from models.demos.deepseek_v3_d_p.tt.v41.flash_sdpa import FlashSdpa +from models.demos.deepseek_v3_d_p.tt.v41.head_layout import o_heads, q_heads +from models.demos.deepseek_v3_d_p.utils.kv_cache_utils import MlaKvCacheFormat + +SENTINEL = 0xFFFFFFFF +ROWS = 32 +PCC_FLOOR = float(os.environ.get("V41_FS_PCC", "0.999")) + + +def _trace_us(mesh_device, fn, calls: int, replays: int) -> float: + for _ in range(2): + ttnn.deallocate(fn()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn() for _ in range(calls)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / calls + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + return round(best * 1e6, 1) + + +def _pcc(a, b) -> float: + a, b = a.double().flatten(), b.double().flatten() + if a.std() == 0 or b.std() == 0: + return float(torch.equal(a, b)) + return float(torch.corrcoef(torch.stack([a, b]))[0, 1]) + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=64 * 1024 * 1024), indirect=True) +def test_v41_flash_sdpa(mesh_device, device_params): + trials = int(os.environ.get("V41_SL_TRIALS", 3)) + replays = int(os.environ.get("V41_SL_REPLAYS", 20)) + calls = int(os.environ.get("V41_SL_CALLS", 16)) + cache_rows = int(os.environ.get("V41_SL_CACHE", 4096)) + width = int(os.environ.get("V41_SL_WIDTH", C.SLIDING_WINDOW + C.INDEX_TOPK)) # the index rows' width + patterns = os.environ.get("V41_FS_PATTERNS", "c1,k2,k3,f8,f16,full").split(",") + sp, tp = tuple(mesh_device.shape) + heads, head_dim, rope_dim, groups = C.NUM_ATTENTION_HEADS, C.HEAD_DIM, C.QK_ROPE_HEAD_DIM, C.O_GROUPS + local_heads, sdpa_heads = heads // tp, max(heads // tp, SDPA_MIN_HEADS) + v_dim = head_dim # the model's V width is the head dim: the scaled rows have no bf16 RoPE tail + scale = head_dim**-0.5 + shard = lambda dims: ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=dims) + replicate = ttnn.ReplicateTensorToMesh(mesh_device) + torch.manual_seed(143) + # the trials' instance copies every partial to its DRAM scratch for the diagnostic; the timing instances + # do not + # (``fs_t`` every row's slots, the test's default; ``fs_c1`` one slot, the decoder's c1-capture layout) + geo = dict(heads=sdpa_heads, rows=ROWS, width=width, v_dim=v_dim, real_heads=local_heads) + fs = FlashSdpa(mesh_device, debug_scratch=True, **geo) + fs_t = FlashSdpa(mesh_device, debug_scratch=False, **geo) + fs_c1 = FlashSdpa(mesh_device, debug_scratch=False, max_live=8, **geo) + + def tiled(t, dims, dtype=ttnn.bfloat16): + return ttnn.from_torch( + t, device=mesh_device, dtype=dtype, layout=ttnn.TILE_LAYOUT, mesh_mapper=shard(dims) if dims else replicate + ) + + def rm(t, dims, dtype=ttnn.bfloat16): + return ttnn.from_torch( + t, + device=mesh_device, + dtype=dtype, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=shard(dims) if dims else replicate, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + ) + + trans = tiled(get_rot_transformation_mat(), None) + sink = (torch.randn(1, 1, 1, heads) * 2.0 / scale).to(torch.bfloat16) + local = sink.view(tp, local_heads) + padded = torch.cat([local, torch.zeros(tp, sdpa_heads - local_heads, dtype=local.dtype)], 1) + sink_local = rm(padded.reshape(1, 1, 1, tp * sdpa_heads), (None, 3)) + + host = {} # the host copies for the diagnostic: the bf16 cache per mesh row, the index rows, the sinks + + def kv_cache(fmt: MlaKvCacheFormat, seed: int): + gen = torch.Generator().manual_seed(seed) + rows = torch.randn(1, 1, sp * cache_rows, head_dim, generator=gen) + if fmt == MlaKvCacheFormat.BF16_RM: + host["cache"] = rows.to(torch.bfloat16).float().view(sp, cache_rows, head_dim) + return rm(rows.to(torch.bfloat16), (2, None)) + latent = ttnn.from_torch( + rows.to(torch.bfloat16), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.ROW_MAJOR_LAYOUT, + mesh_mapper=shard((2, None)), + ) + fp8, scales = ttnn.experimental.deepseek_prefill.per_token_cast_to_fp8(latent, round_scale_to_power_of_two=True) + packed = ttnn.experimental.deepseek_prefill.pack_scaled_fp8_kv_cache(fp8, scales) + for t in (latent, fp8, scales): + ttnn.deallocate(t) + return packed + + def index_rows(pattern: str, seed: int): + """``[1, 1, 32, width]`` uint32 per mesh row and the live rows per mesh row: ``c1`` one live row (a + random + one, every key), ``kK`` the live row and K draft rows (rows 28.. as the verify step's c1 layout: each + draft + row sees the user's keys and the drafts before it, at a random valid count), ``fK`` the first K rows + live + and ``full`` every row live, with random valid counts (a sentinel tail).""" + gen = torch.Generator().manual_seed(seed) + rows = torch.full((1, sp, ROWS, width), SENTINEL, dtype=torch.int64) + live_rows = [] + u = int(torch.randint(ROWS - 4, (1,), generator=gen)) # the same rows on every mesh row (one live page) + for mesh_row in range(sp): + if pattern == "c1": + live = [u] + elif pattern.startswith("k"): + live = [u] + list(range(ROWS - 4, ROWS - 4 + int(pattern[1:]))) + elif pattern != "full" and pattern.startswith("f"): + live = list(range(int(pattern[1:]))) # the first K rows live, random valid counts + else: + live = list(range(ROWS)) + keys = torch.randperm(cache_rows, generator=gen) + for j, r in enumerate(live): + if pattern == "c1": + n = width + elif pattern.startswith("k"): + n = int(torch.randint(width // 2, width + 1 - 4, (1,), generator=gen)) + j + rows[0, mesh_row, r, :n] = keys[:n] + continue + else: + n = int(torch.randint(1, width + 1, (1,), generator=gen)) + rows[0, mesh_row, r, :n] = torch.randperm(cache_rows, generator=gen)[:n] + live_rows.append(live) + host["rows"] = rows[0] + return rm(rows.to(torch.int32), (1, None), ttnn.uint32), live_rows, fs.live_page(live_rows[0]) + + def diagnose(qh, out_flash, out_sparse, live_r, chip: int = 0): + """The bf16 case on one chip against a torch reference, per live row: the flash and sparse outputs + against + torch, and the partials read back from the scratch (each unit's max, sum tile and acc against the same + arithmetic in torch, then the merge of the device partials in torch against the flash output) -> a + list of + per-row dicts.""" + qt = ttnn.to_torch(ttnn.get_device_tensors(qh)[chip]).float()[0] # [32 heads, 32 rows, 512] + fl_all = ttnn.to_torch(ttnn.get_device_tensors(out_flash)[chip]).float()[0] + sr_all = ttnn.to_torch(ttnn.get_device_tensors(out_sparse)[chip]).float()[0] + scratch = ttnn.to_torch(ttnn.get_device_tensors(fs.scratch)[chip]).float()[0, 0] # [tiles * 32, 32] + sink_t = padded[chip % tp].float() * scale # sparse_sdpa scales the sink as it scales the scores + out = [] + for i, u in enumerate(live_r): + q = qt[:, u, :] + idx = host["rows"][chip // tp, u] + nv = int((idx != SENTINEL).sum()) + keys = host["cache"][chip // tp][idx[:nv].long()] # [nv, 512] + scores = q @ keys.T # [32, nv] + p = torch.softmax(torch.cat([scores * scale, sink_t[:, None]], 1), dim=1)[:, :nv] + ref = p @ keys + fl, sr = fl_all[:, u, :], sr_all[:, u, :] + d = {"row": u, "nv": nv, "torch_vs_flash": round(_pcc(ref[:local_heads], fl[:local_heads]), 5)} + d["torch_vs_sparse"] = round(_pcc(ref[:local_heads], sr[:local_heads]), 5) + U = -(-nv // fs.kpu) + pm, ps, pa, ms = [], [], [], [] + for j in range(U): + base = (i * fs.umax + j) * fs.slot_tiles + tile = lambda t: scratch[(base + t) * 32 : (base + t + 1) * 32] + s_j = scores[:, j * fs.kpu : (j + 1) * fs.kpu] + m_ref = s_j.max(dim=1).values + p_ref = torch.exp((s_j - m_ref[:, None]) * scale) + a_ref = p_ref @ keys[j * fs.kpu : (j + 1) * fs.kpu] + pm.append(_pcc(m_ref[:local_heads], tile(0)[:local_heads, 0])) + p_dev = tile(1)[:local_heads, : s_j.shape[1]] + ps.append( + _pcc(p_ref[:local_heads], p_dev) + if s_j.shape[1] > 1 + else float((p_ref[:local_heads] - p_dev).abs().max() < 0.02) # one key: exp(0) = 1 exactly + ) + acc = torch.cat([tile(2 + v) for v in range(fs.vt)], 1) + pa.append(_pcc(a_ref[:local_heads], acc[:local_heads])) + ms.append(tile(0)[:, 0]) + d["units"] = U + d["partial_pcc_min"] = [round(min(pm), 4), round(min(ps), 4), round(min(pa), 4)] + d["bad_units"] = [j for j in range(U) if min(pm[j], ps[j], pa[j]) < 0.99] + ms = torch.stack(ms) + M = ms.max(dim=0).values + w = torch.exp((ms - M[None]) * scale) + num = torch.zeros(32, v_dim) + den = torch.exp((sink_t / scale - M) * scale) + for j in range(U): + base = (i * fs.umax + j) * fs.slot_tiles + acc = torch.cat([scratch[(base + 2 + v) * 32 : (base + 3 + v) * 32] for v in range(fs.vt)], 1) + num += w[j][:, None] * acc + den += w[j] * scratch[(base + 1) * 32 : (base + 2) * 32].sum(dim=1) + hm = num / den[:, None] + d["merge_of_partials_vs_flash"] = round(_pcc(hm[:local_heads], fl[:local_heads]), 5) + d["merge_of_partials_vs_torch"] = round(_pcc(hm[:local_heads], ref[:local_heads]), 5) + out.append(d) + return out + + def queries(seed: int): + gen = torch.Generator().manual_seed(seed) + q = torch.randn(1, 1, sp * ROWS, heads * head_dim, generator=gen) * 2.0 + angle = torch.rand(sp * ROWS, rope_dim // 2, generator=gen) * 6.2832 + angle = torch.cat([angle, angle], -1) + cos = tiled(angle.cos().view(1, 1, sp * ROWS, rope_dim).to(torch.bfloat16), (2, None)) + sin = tiled(angle.sin().view(1, 1, sp * ROWS, rope_dim).to(torch.bfloat16), (2, None)) + return tiled(q.to(torch.bfloat16), (2, 3)), cos, sin + + def sparse(qh, kv, rows, fmt, w, live=None): # ``live`` is the flash program's; the timing calls pass it to both + return ttnn.transformer.sparse_sdpa( + qh, + kv, + rows, + head_dim, + kv_format=fmt.sparse_sdpa_format, + scale=scale, + k_chunk_size=next(c for c in (128, 64, 32) if w % c == 0), + attention_sink=sink_local, + ) + + def flash(qh, kv, rows, fmt, w, live=None, inst=None): + return (inst or fs)(qh, kv, rows, fmt == MlaKvCacheFormat.SCALED_FP8, scale, sink_local, live=live) + + result = { + "mesh": [sp, tp], + "width": width, + "cache_rows": cache_rows, + "kpu": fs.kpu, + "nsem": [fs.nsem, fs_c1.nsem], + "trials": [], + } + caches = {fmt.name: kv_cache(fmt, 7) for fmt in (MlaKvCacheFormat.BF16_RM, MlaKvCacheFormat.SCALED_FP8)} + failures = [] + for trial in range(trials): + q, cos, sin = queries(100 + trial) + qh = q_heads(q, cos, sin, trans, local_heads, rope_dim, out_heads=sdpa_heads) + for fmt in (MlaKvCacheFormat.BF16_RM, MlaKvCacheFormat.SCALED_FP8): + for pattern in patterns: + rows, live_rows, live = index_rows(pattern, 200 + trial) + ref = sparse(qh, caches[fmt.name], rows, fmt, width) + out = flash(qh, caches[fmt.name], rows, fmt, width, live) + expect = (1, sdpa_heads, ROWS, v_dim) + assert tuple(out.shape) == tuple(ref.shape) == expect, (tuple(out.shape), tuple(ref.shape)) + words, max_abs, ref_max, pcc_min, dead_nonzero = 0, 0.0, 0.0, 1.0, 0 + for chip, (a, b) in enumerate(zip(ttnn.get_device_tensors(ref), ttnn.get_device_tensors(out))): + a, b = ttnn.to_torch(a).float(), ttnn.to_torch(b).float() + live_r = live_rows[chip // tp] + dead = [r for r in range(ROWS) if r not in live_r] + al, bl = a[:, :local_heads, live_r, :], b[:, :local_heads, live_r, :] + words += int((al != bl).sum()) + max_abs = max(max_abs, float((al - bl).abs().max())) + ref_max = max(ref_max, float(al.abs().max())) + pcc_min = min(pcc_min, _pcc(al, bl)) + if dead: + dead_nonzero += int((b[:, :, dead, :] != 0).sum()) + entry = { + "trial": trial, + "kv_format": fmt.name, + "pattern": pattern, + "live_rows": len(live_rows[0]), + "words_differ": words, + "max_abs": max_abs, + "rel_max": max_abs / max(ref_max, 1e-30), + "pcc_min": pcc_min, + "dead_nonzero": dead_nonzero, + } + if fmt == MlaKvCacheFormat.BF16_RM and trial == 0 and os.environ.get("V41_FS_DIAG", "1") == "1": + entry["diag"] = diagnose(qh, out, ref, live_rows[0]) + result["trials"].append(entry) + logger.info(f"flash sdpa trial {json.dumps(entry)}") + if pcc_min < PCC_FLOOR or dead_nonzero: + failures.append(entry) + for t in (rows, ref, out, live): + ttnn.deallocate(t) + for t in (q, cos, sin, qh): + ttnn.deallocate(t) + # the traced per-layer time of each SDPA alone per pattern and format, and the local chain for c1 + q, cos, sin = queries(999) + qh = q_heads(q, cos, sin, trans, local_heads, rope_dim, out_heads=sdpa_heads) + timing = {} + for fmt in (MlaKvCacheFormat.SCALED_FP8, MlaKvCacheFormat.BF16_RM): + for pattern in patterns: + rows, _, live = index_rows(pattern, 999) + timed = [("sparse", sparse), ("flash", lambda *a: flash(*a, inst=fs_t))] + if pattern in ("c1", "k2", "k3", "f8"): + timed.append(("flash1", lambda *a: flash(*a, inst=fs_c1))) # the one-slot (served) layout + for name, fn in timed: + key = f"{name}_{fmt.name}_{pattern}" + timing[key] = _trace_us( + mesh_device, lambda: fn(qh, caches[fmt.name], rows, fmt, width, live), calls, replays + ) + logger.info(f"flash sdpa traced {key}: {timing[key]} us a layer") + if pattern == "c1" and fmt == MlaKvCacheFormat.SCALED_FP8: + for name, fn in (("sparse", sparse), ("flash", lambda *a: flash(*a, inst=fs_c1))): + chain = lambda: o_heads( + fn( + q_heads(q, cos, sin, trans, local_heads, rope_dim, out_heads=sdpa_heads), + caches[fmt.name], + rows, + fmt, + width, + live, + ), + cos, + ttnn.neg(sin), + trans, + groups // tp, + rope_dim, + heads=local_heads, + ) + timing[f"chain_{name}_c1"] = _trace_us(mesh_device, chain, calls, replays) + logger.info(f"flash sdpa traced chain {name} c1: {timing[f'chain_{name}_c1']} us a layer") + for t in (rows, live): + ttnn.deallocate(t) + # the window layers' geometry: bf16 rows, 128 keys in every row (the pool's window rows are constants: + # every + # row of the tile reads its user's ring), one live row + if width > 128: + rows_w = torch.zeros((1, sp, ROWS, 128), dtype=torch.int64) + for mesh_row in range(sp): + for r in range(ROWS): + rows_w[0, mesh_row, r, :] = torch.randperm(cache_rows)[:128] + rows = rm(rows_w.to(torch.int32), (1, None), ttnn.uint32) + live = fs.live_page([5]) + fmt = MlaKvCacheFormat.BF16_RM + for name, fn in (("sparse", sparse), ("flash", lambda *a: flash(*a, inst=fs_c1))): + key = f"{name}_{fmt.name}_c1_w128" + timing[key] = _trace_us(mesh_device, lambda: fn(qh, caches[fmt.name], rows, fmt, 128, live), calls, replays) + logger.info(f"flash sdpa traced {key}: {timing[key]} us a layer") + for t in (rows, live): + ttnn.deallocate(t) + result["layer_us"] = timing + logger.info(f"V41_FS_RESULT {json.dumps(result)}") + out_path = os.environ.get("V41_SL_OUT") + if out_path: + with open(os.path.expanduser(out_path), "w") as f: + json.dump(result, f, indent=1) + assert not failures, failures diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_generator_vllm.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_generator_vllm.py new file mode 100644 index 000000000000..45c46085940a --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_generator_vllm.py @@ -0,0 +1,286 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The vllm-tt-plugin contract of ``DeepseekV41ForCausalLM`` on the Galaxy, driven as the plugin's model +runner +would, without vLLM: build (``initialize_vllm_model``), ``allocate_kv_cache``, the two-phase warm-up, a +two-request +prefill batch, decode steps on the batch, then a third request that takes a slot a finished request held (the +captures made on the slot serve the new request, an odd prompt whose compressor carry must reach the odd +capture). + +Request 0 is the text the model knows by heart (A Tale of Two Cities), teacher-forced on its next tokens and +judged +by ``book_top1`` as the decode test; request 1 is a different stretch of the same text, free-running, present +so the +batch has two rows. ``V41_GEN_STEPS`` decode steps (default 16), ``V41_GEN_PROMPT`` tokens (255), +``V41_GEN_LAYERS`` +(``all``; the generator reads it as ``TT_V41_LAYERS``). ``V41_GEN_TRACE_PREFILL=1`` lists the prompt length in +``TT_V41_TRACE_PREFILL_LENGTHS`` so every request's prefill is a replay of a capture made at warm-up (the odd +prompt's +compressor carry then comes out of the replay); the trace region's bytes are recorded. ``V41_GEN_MAX_SEQ`` +raises the +context to a served value (the 1M catalog's 1,053,696; the trace region then defaults to 2 GiB) and +``V41_GEN_SHORT_PROMPT`` (e.g. 1,028) adds a fourth request of that length, listed with the prompt, which must +replay +on the short chunk's state when ``TT_V41_SHORT_CHUNK`` is set (``book_top1_short``). Results go +to the log as ``V41_GENERATOR_RESULT {json}`` and to ``V41_GEN_OUT``. +""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_decode import TRACE_REGION +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_depth import _dram_bytes, parse_layers +from models.demos.deepseek_v3_d_p.tt.v41.generator_vllm import DeepseekV41ForCausalLM +from models.demos.deepseek_v3_d_p.tt.v41.weights import resolve_checkpoint + + +def _trace_bytes(mesh_device) -> int | None: + """Per-chip bytes allocated in the trace region (the captures), as ``_dram_bytes`` for DRAM.""" + try: + view = ttnn.get_memory_view(mesh_device, ttnn.BufferType.TRACE) + return int(view.total_bytes_allocated_per_bank * view.num_banks) + except Exception as e: + logger.warning(f"trace region statistics unavailable: {e}") + return None + + +def _page_table(first_blocks: list[int], width: int = 8) -> torch.Tensor: + table = torch.zeros(len(first_blocks), width, dtype=torch.int32) + for i, b in enumerate(first_blocks): + table[i, 0] = b + return table + + +def _trace_region() -> int: + """The decode test's region, or the served 2 GiB when ``V41_GEN_MAX_SEQ`` is past the first score bucket.""" + return TRACE_REGION if int(os.environ.get("V41_GEN_MAX_SEQ", 0)) <= 16384 else 2 * 1024**3 + + +@pytest.mark.timeout(int(os.environ.get("V41_GEN_TIMEOUT", 2 * 3600))) +@pytest.mark.parametrize( + "mesh_device, device_params", + galaxy_meshes(trace_region_size=int(os.environ.get("V41_GEN_TRACE_REGION", _trace_region()))), + indirect=True, +) +def test_v41_generator_vllm(mesh_device, device_params, monkeypatch): + layers = parse_layers(os.environ.get("V41_GEN_LAYERS", "all")) + prompt = int(os.environ.get("V41_GEN_PROMPT", 255)) + steps = int(os.environ.get("V41_GEN_STEPS", 16)) + chunk = int(os.environ.get("TT_V41_CHUNK", 1024)) + monkeypatch.setenv("TT_V41_CHUNK", str(chunk)) + monkeypatch.setenv("TT_V41_LAYERS", ",".join(str(l) for l in layers)) + trace_prefill = os.environ.get("V41_GEN_TRACE_PREFILL", "0") == "1" + # a fourth request of this length (0: none), listed with the prompt: with TT_V41_SHORT_CHUNK set it must + # replay + # on the short chunk's state + short_prompt = int(os.environ.get("V41_GEN_SHORT_PROMPT", 0)) + if trace_prefill: + monkeypatch.setenv( + "TT_V41_TRACE_PREFILL_LENGTHS", str(prompt) if not short_prompt else f"{prompt},{short_prompt}" + ) + elif "TT_V41_TRACE_PREFILL_LENGTHS" not in os.environ: # an explicit list in the environment is kept + monkeypatch.setenv("TT_V41_TRACE_PREFILL_LENGTHS", "") + if resolve_checkpoint() is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + # the context: what the prompts need, or the served value (V41_GEN_MAX_SEQ, e.g. the 1M catalog's 1053696) + max_seq_len = max(prompt + steps + 64, int(os.environ.get("V41_GEN_MAX_SEQ", 0))) + result = { + "layers": layers, + "n_layers": len(layers), + "prompt": prompt, + "steps": steps, + "chunk": chunk, + "max_model_len": max_seq_len, + "mesh": list(mesh_device.shape), + "trace_prefill": trace_prefill, + } + + t0 = time.time() + batch = int(os.environ.get("V41_GEN_BATCH", 8)) # decode slots: a multiple of the mesh rows (the pool) + model = DeepseekV41ForCausalLM.initialize_vllm_model( + None, mesh_device, max_batch_size=batch, max_seq_len=max_seq_len + ) + result["build_s"] = round(time.time() - t0, 1) + kv = model.allocate_kv_cache((16, 1, 64, 512), torch.bfloat16, len(layers)) + assert len(kv) == len(layers) + t0 = time.time() + model.warmup_model_prefill(enable_trace=False, kv_cache=kv, can_sample_on_device=False) + model.warmup_model_decode( + enable_trace=False, kv_cache=kv, max_batch_size=batch, num_blocks=16, can_sample_on_device=False + ) + model.warmup_model_prefill(enable_trace=True, kv_cache=kv, can_sample_on_device=False) + model.warmup_model_decode( + enable_trace=True, kv_cache=kv, max_batch_size=batch, num_blocks=16, can_sample_on_device=False + ) + result["warmup_s"] = round(time.time() - t0, 1) + result["dram_after_warmup_bytes"] = _dram_bytes(mesh_device) + result["trace_after_warmup_bytes"] = _trace_bytes(mesh_device) + result["prefill_traces"] = len(model.prefill_traces) + logger.info( + f"generator: built in {result['build_s']} s, warmed up in {result['warmup_s']} s; " + f"{result['prefill_traces']} prefill traces, trace region {result['trace_after_warmup_bytes']} B" + ) + + need = 2 * prompt + 2 * steps + 2 + try: + text = orc.text_tokens(need)[0] + except ValueError as e: # a prompt beyond the book (the long points): the book repeated end to end + have = int(str(e).split()[0]) + book = orc.text_tokens(have)[0] + text = book.repeat(-(-need // have))[:need] + logger.info(f"generator: {have} tokens of text repeated {-(-need // have)} times for prompts of {prompt}") + book_a, other = text[: prompt + steps + 1], text[prompt + steps + 1 :] + tokenizer = model.tokenizer + + chunked = os.environ.get("V41_GEN_CHUNKED", "0") == "1" # drive each prompt chunk by chunk, as the plugin + result["chunked"] = chunked + + def prefill_call(rows: list[tuple[int, torch.Tensor]], starts: list[int], ends: list[int] | None = None): + """One ``prefill_forward``: per row the whole prompt, the chunk's start and (chunked) its end, as the + plugin passes them (``prompt_total_lens`` with the whole prompt).""" + width = max(int(t.numel()) for _, t in rows) + tokens = torch.zeros(len(rows), width, dtype=torch.int32) + for i, (_, t) in enumerate(rows): + tokens[i, : t.numel()] = t + totals = [int(t.numel()) for _, t in rows] + out = model.prefill_forward( + tokens=tokens, + prompt_lens=torch.tensor(totals if ends is None else ends), + page_table=_page_table([b for b, _ in rows]), + start_pos=torch.tensor(starts, dtype=torch.int32), + kv_cache=kv, + enable_trace=True, + **({} if ends is None else {"prompt_total_lens": torch.tensor(totals)}), + ) + assert out.shape == (len(rows), 1, C.VOCAB_SIZE) and not torch.isnan(out).any() + return out + + def prefill(rows: list[tuple[int, torch.Tensor]]) -> torch.Tensor: + t0 = time.time() + if not chunked: + out = prefill_call(rows, [0] * len(rows)) + else: + # the plugin's chunked prefill with aligned chunks: one request at a time, each chunk a call with + # the + # prefix so far (the scheduler admits one prefill a step and advances a partial prefill alone) + out = torch.zeros(len(rows), 1, C.VOCAB_SIZE) + for i, (block, t) in enumerate(rows): + n = int(t.numel()) + for start in range(0, n, chunk): + end = min(start + chunk, n) + out[i] = prefill_call([(block, t)], [start], [end])[0] + result.setdefault("prefill_s", []).append(round(time.time() - t0, 3)) + return out + + sample_on_device = DeepseekV41ForCausalLM.SAMPLE_ON_DEVICE + result["sample_on_device"] = sample_on_device + + class _Params: # the plugin's TTSamplingParams, as lists per row: argmax (temperature 0) on every row + def __init__(self, n): + self.temperature, self.top_k, self.top_p = [0.0] * n, [1] * n, [0.0] * n + self.repetition_penalty = 1.0 + + def decode(rows: list[tuple[int, int, int]]) -> torch.Tensor: + """rows: (first block, token, position); block 0 rows are idle -> fp32 logits [rows, 1, vocab] (a + one-hot of the sampled token when sampling on device: the callers take the argmax).""" + t0 = time.time() + out = model.decode_forward( + tokens=torch.tensor([[t] for _, t, _ in rows], dtype=torch.int32), + start_pos=torch.tensor([p for _, _, p in rows], dtype=torch.int32), + page_table=_page_table([b for b, _, _ in rows]), + kv_cache=kv, + enable_trace=True, + read_from_device=True, + sampling_params=_Params(len(rows)) if sample_on_device else None, + ) + result.setdefault("decode_s", []).append(round(time.time() - t0, 3)) + result.setdefault("decode_last", []).append(dict(model.decoder.last)) + if sample_on_device: + assert out.shape == (len(rows),), tuple(out.shape) + ids = out + out = torch.zeros(len(rows), 1, C.VOCAB_SIZE) + for i in range(len(rows)): + out[i, 0, int(ids[i])] = 1.0 + assert out.shape == (len(rows), 1, C.VOCAB_SIZE) and not torch.isnan(out).any() + return out + + # --- two requests: the book (block 7, teacher-forced) and another stretch (block 9, free-running) --- + logits = prefill([(7, book_a[:prompt]), (9, other[:prompt])]) + fed_a, tok_b = int(book_a[prompt]), int(logits[1, 0].argmax()) + hits, preds_a, text_b = 0, [], [] + for s in range(steps): + p = prompt + s + out = decode([(7, fed_a, p), (9, tok_b, p)]) + pred_a = int(out[0, 0].argmax()) + preds_a.append(pred_a) + hits += pred_a == int(book_a[p + 1]) + fed_a = int(book_a[p + 1]) + tok_b = int(out[1, 0].argmax()) + text_b.append(tok_b) + logger.info(f"generator: step {s} at {p}: book {pred_a} vs {int(book_a[p + 1])}, other {tok_b}") + result["book_top1"] = round(hits / steps, 4) + result["text"] = tokenizer.decode(preds_a) + result["book_text"] = tokenizer.decode(book_a[prompt + 1 : prompt + steps + 1].tolist()) + result["other_text"] = tokenizer.decode(text_b) + + # --- a third request (a free slot, or the least recently used one when V41_GEN_BATCH is 2 per 1 mesh + # row): + # the same book prompt, from the start --- + logits = prefill([(11, book_a[:prompt])]) + fed, hits2, preds = int(book_a[prompt]), 0, [] + for s in range(min(steps, 8)): + p = prompt + s + out = decode([(11, fed, p), (0, 0, 0)]) + pred = int(out[0, 0].argmax()) + preds.append(pred) + hits2 += pred == int(book_a[p + 1]) + fed = int(book_a[p + 1]) + result["book_top1_reused_slot"] = round(hits2 / min(steps, 8), 4) + result["text_reused_slot"] = tokenizer.decode(preds) + + # --- a fourth request of V41_GEN_SHORT_PROMPT tokens (the book from the start), on the short chunk's + # state when + # one is configured: the prefill must be a replay on that chunk, and the teacher-forced decode is judged + # as above --- + if short_prompt: + assert short_prompt + 9 <= book_a.numel(), (short_prompt, book_a.numel()) + logits = prefill([(13, book_a[:short_prompt])]) + result["short_prefill_last"] = dict(model.prefill_last) + if model.short: + assert model.prefill_last["chunk"] == model.short[0][0], model.prefill_last + assert model.prefill_last["mode"] == "replay", model.prefill_last + fed, hits3, preds = int(book_a[short_prompt]), 0, [] + for s in range(min(steps, 8)): + p = short_prompt + s + out = decode([(13, fed, p), (0, 0, 0)]) + pred = int(out[0, 0].argmax()) + preds.append(pred) + hits3 += pred == int(book_a[p + 1]) + fed = int(book_a[p + 1]) + result["book_top1_short"] = round(hits3 / min(steps, 8), 4) + result["text_short"] = tokenizer.decode(preds) + result["book_text_short"] = tokenizer.decode( + book_a[short_prompt + 1 : short_prompt + min(steps, 8) + 1].tolist() + ) + result["slots"] = sorted(model.slots.keys()) + result["dram_after_requests_bytes"] = _dram_bytes(mesh_device) + logger.info("V41_GENERATOR_RESULT " + json.dumps(result)) + out = os.environ.get("V41_GEN_OUT") + if out: + with open(out, "w") as f: + json.dump(result, f, indent=1) + for trace in model.prefill_traces.values(): + trace.release() + model.decoder.release() diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_head_layout.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_head_layout.py new file mode 100644 index 000000000000..62109a455125 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_head_layout.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Fused head layout of the V4.1 attention (tt/v41/head_layout.py, bead 8y7.9.9) vs the composite ttnn path. + +``q_heads`` replaces ``nlp_create_qkv_heads`` + the RoPE tail (slice, ``rotary_embedding_llama``, untilize, +``slice_write``); ``o_heads`` replaces the inverse RoPE tail (slice, tilize, rotate, untilize, ``slice_write``, +tilize) + ``nlp_concat_heads`` of the head groups. Both run the rotation's op sequence with the same bf16 +intermediates, so they must match the composite path bit for bit. ``test_head_layout_traced_time`` logs the traced +per-call time of both paths at the production per-chip shape (the sequence shard of chunk 5120 on 2x4: 640 tokens, +64 heads of 512) as ``V41_HEADS_PERF`` JSON lines. +""" + +import json +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.tt.mla.rope import get_rot_transformation_mat +from models.demos.deepseek_v3_d_p.tt.v41.head_layout import o_heads, q_heads + +ROPE = 64 +REPLAYS = 20 +# (tokens, heads, head_dim, groups): small = SmallV41Config on one chip of 2x4 (32 heads of 128, 8 groups) +SHAPES = {"small": (64, 32, 128, 8), "tiny": (32, 8, 128, 2), "production": (640, 64, 512, 8)} + + +def _upload(device, t, layout=ttnn.TILE_LAYOUT): + return ttnn.from_torch(t.to(torch.bfloat16), dtype=ttnn.bfloat16, layout=layout, device=device) + + +def _inputs(device, shape, seed=0): + tokens, heads, head_dim, _ = shape + gen = torch.Generator().manual_seed(seed) + angle = torch.rand(tokens, ROPE // 2, generator=gen) * 6.3 + # interleaved pairs, as the V4.1 RoPE tables (rotary_embedding_llama's trans_mat convention) + cos = torch.cos(angle).repeat_interleave(2, dim=-1).reshape(1, 1, tokens, ROPE) + sin = torch.sin(angle).repeat_interleave(2, dim=-1).reshape(1, 1, tokens, ROPE) + return dict( + q=_upload(device, torch.randn(1, 1, tokens, heads * head_dim, generator=gen)), + attn=_upload(device, torch.randn(1, heads, tokens, head_dim, generator=gen), ttnn.ROW_MAJOR_LAYOUT), + cos=_upload(device, cos), + sin=_upload(device, sin), + trans=_upload(device, get_rot_transformation_mat()), + ) + + +def _composite_q(d, heads): + """tt/v41/attention.py before 8y7.9.9: nlp_create_qkv_heads, then ``_rope_tail``.""" + q, _, _ = ttnn.experimental.nlp_create_qkv_heads(d["q"], num_heads=heads, num_kv_heads=0, transpose_k_heads=False) + b, h, s, dim = q.shape + tail = ttnn.slice(q, [0, 0, 0, dim - ROPE], [b, h, s, dim]) + tail = ttnn.experimental.rotary_embedding_llama(tail, d["cos"], d["sin"], d["trans"], is_decode_mode=False) + out = ttnn.to_layout(q, ttnn.ROW_MAJOR_LAYOUT) + ttnn.experimental.slice_write( + ttnn.to_layout(tail, ttnn.ROW_MAJOR_LAYOUT), out, [0, 0, 0, dim - ROPE], [b, h, s, dim], [1, 1, 1, 1] + ) + return out + + +def _composite_o(d, groups): + """tt/v41/attention.py before 8y7.9.9: ``_inverse_rope_tail``, then the groups' nlp_concat_heads.""" + t = ttnn.clone(d["attn"]) # the composite path rotates in place + b, h, s, dim = t.shape + tail = ttnn.to_layout(ttnn.slice(t, [0, 0, 0, dim - ROPE], [b, h, s, dim]), ttnn.TILE_LAYOUT) + tail = ttnn.experimental.rotary_embedding_llama( + tail, d["cos"], ttnn.neg(d["sin"]), d["trans"], is_decode_mode=False + ) + ttnn.experimental.slice_write( + ttnn.to_layout(tail, ttnn.ROW_MAJOR_LAYOUT), t, [0, 0, 0, dim - ROPE], [b, h, s, dim], [1, 1, 1, 1] + ) + t = ttnn.reshape(ttnn.to_layout(t, ttnn.TILE_LAYOUT), [groups, h // groups, s, dim]) + return ttnn.reshape(ttnn.experimental.nlp_concat_heads(t), [1, groups, s, h // groups * dim]) + + +def _fused_q(d, heads): + return q_heads(d["q"], d["cos"], d["sin"], d["trans"], heads, ROPE) + + +def _fused_o(d, groups): + return o_heads(d["attn"], d["cos"], ttnn.neg(d["sin"]), d["trans"], groups, ROPE) + + +def _torch_q(d, heads): + """fp32 torch of the same map (interleaved-pair RoPE on each head's tail) for a coarse semantic check.""" + q = ttnn.to_torch(d["q"]).float() + s = q.shape[-2] + q = q.reshape(s, heads, -1).transpose(0, 1)[None] + cos, sin = (ttnn.to_torch(d[k]).float()[0, 0] for k in ("cos", "sin")) + tail = q[..., -ROPE:] + rot = torch.stack([-tail[..., 1::2], tail[..., 0::2]], dim=-1).flatten(-2) + return torch.cat([q[..., :-ROPE], tail * cos + rot * sin], dim=-1) + + +@pytest.mark.parametrize("shape", list(SHAPES)) +@pytest.mark.parametrize("side", ["q", "o"]) +def test_head_layout_matches_composite(device, side, shape): + tokens, heads, head_dim, groups = SHAPES[shape] + d = _inputs(device, SHAPES[shape]) + t0 = time.perf_counter() + fused_fn, composite_fn, arg = (_fused_q, _composite_q, heads) if side == "q" else (_fused_o, _composite_o, groups) + fused = [fused_fn(d, arg) for _ in range(2)] + logger.info(f"{side} {shape}: 2 fused calls {time.perf_counter() - t0:.2f}s (incl. compile)") + want_layout = ttnn.ROW_MAJOR_LAYOUT if side == "q" else ttnn.TILE_LAYOUT + want_shape = (1, heads, tokens, head_dim) if side == "q" else (1, groups, tokens, heads // groups * head_dim) + assert fused[0].layout == want_layout and tuple(fused[0].shape) == want_shape, (fused[0].layout, fused[0].shape) + got = [ttnn.to_torch(f) for f in fused] + composite = ttnn.to_torch(composite_fn(d, arg)) + assert torch.equal(got[0], got[1]), "fused head layout is not deterministic" + mismatch = (got[0] != composite).sum().item() + logger.info(f"{side} {shape}: elements != composite {mismatch} of {composite.numel()}") + if side == "q": + ref = _torch_q(d, heads) + err = (got[0].float() - ref).abs().max().item() + logger.info(f"q {shape}: max |fused - fp32 torch| {err:.3e}") + assert err < 0.05, err # bf16 intermediates of |x| <~ 5 + assert mismatch == 0, f"{mismatch} elements differ from the composite path" + + +def _traced_us(device, fn): + fn() + ttnn.synchronize_device(device) + tid = ttnn.begin_trace_capture(device, cq_id=0) + out = fn() + ttnn.end_trace_capture(device, tid, cq_id=0) + ttnn.execute_trace(device, tid, cq_id=0, blocking=True) + t0 = time.perf_counter() + for _ in range(REPLAYS): + ttnn.execute_trace(device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(device) + us = (time.perf_counter() - t0) / REPLAYS * 1e6 + ttnn.release_trace(device, tid) + out.deallocate() + return us + + +@pytest.mark.parametrize("impl", ["fused", "composite"]) +@pytest.mark.parametrize("side", ["q", "o"]) +@pytest.mark.parametrize("device_params", [{"trace_region_size": 32 << 20}], indirect=True) +def test_head_layout_traced_time(device, side, impl): + tokens, heads, head_dim, groups = SHAPES["production"] + d = _inputs(device, SHAPES["production"]) + fns = {("q", "fused"): _fused_q, ("q", "composite"): _composite_q} + fns |= {("o", "fused"): _fused_o, ("o", "composite"): _composite_o} + arg = heads if side == "q" else groups + us = _traced_us(device, lambda: fns[side, impl](d, arg)) + moved = 2 * 2 * tokens * heads * head_dim # read + write the heads once, bf16 + bound_us = moved / (512.0 * 1e3) # Blackhole p150 DRAM peak + record = dict(side=side, impl=impl, tokens=tokens, heads=heads, head_dim=head_dim, traced_us=round(us, 1)) + record |= dict(dram_bound_us=round(bound_us, 1), dram_util=round(bound_us / us, 3)) + logger.info("V41_HEADS_PERF " + json.dumps(record)) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_host_copy_bench.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_host_copy_bench.py new file mode 100644 index 000000000000..6da4dbae4006 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_host_copy_bench.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The host-to-device cost of the decode step's small inputs. ``test_v41_host_copy_bench``: +for each dtype / layout / shape variant the median and p90 of 60 ``copy_host_to_device_tensor`` into a fixed +mesh +buffer (one shard per rank, replicated over TP, as the step stages them), of the host tensor's construction +(``from_torch`` with the mesh mapper, or ``ttnn.Tensor`` per rank + ``from_host_shards``) and of a device +sync. +``test_v41_first_copy_after_step``: the step's sequence (a captured trace replayed non-blocking, a device +sync, a +read of a device tensor, then the copies) and the cost of the first copy after it against the second, with the +replay, the sync and the read left out one at a time (the served step's first copy costs 1.7 to 2.0 ms, the +next +ones 0.2 to 0.3, the isolated copy 0.09). No weights; ~2 minutes on the Galaxy. ``V41_COPY_BENCH_OUT``: the +JSON.""" + +import json +import os +import statistics +import time + +import pytest +import torch + +import ttnn +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes + +VARIANTS = [ + # name, torch dtype, ttnn dtype, layout, per-rank shape + ("ints rm u32 [14,32]", torch.int32, ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 14, 32)), + ("one rm u32 [1,12800]", torch.int32, ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 1, 12800)), + ("floats rm bf16 [2,32]", torch.float32, ttnn.bfloat16, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 2, 32)), + ("rope tile bf16 [32,384]", torch.float32, ttnn.bfloat16, ttnn.TILE_LAYOUT, (1, 1, 32, 384)), + ("one tile u32 [64,384]", torch.int32, ttnn.uint32, ttnn.TILE_LAYOUT, (1, 1, 64, 384)), + ("one tile u32 [32,416]", torch.int32, ttnn.uint32, ttnn.TILE_LAYOUT, (1, 1, 32, 416)), + ("ints rm u32 [1,448]", torch.int32, ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 1, 448)), + ("ints rm u16 [14,32]", torch.int16, ttnn.uint16, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 14, 32)), + ("ints tile u32 [32,32]", torch.int32, ttnn.uint32, ttnn.TILE_LAYOUT, (1, 1, 32, 32)), + ("engram rm u16 [192,160]", torch.int16, ttnn.uint16, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 192, 160)), +] + + +def _stats(samples: list[float]) -> dict: + s = sorted(samples) + return {"median_us": 1e6 * statistics.median(s), "p90_us": 1e6 * s[int(0.9 * (len(s) - 1))], "n": len(s)} + + +def _dump(name: str, out: dict) -> None: + path = os.environ.get("V41_COPY_BENCH_OUT") + if path: + base, ext = os.path.splitext(path) + json.dump(out, open(f"{base}.{name}{ext}", "w"), indent=1) + + +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(), indirect=True) +def test_v41_host_copy_bench(mesh_device, device_params): + sp, tp = tuple(mesh_device.shape) + mapper = ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, None)) + out = {} + for name, tdtype, dtype, layout, shape in VARIANTS: + full = (sp,) + shape[1:] + host_t = torch.zeros(full, dtype=tdtype) + + def via_from_torch(): + return ttnn.from_torch(host_t, dtype=dtype, layout=layout, mesh_mapper=mapper) + + def via_shards(): + parts = [] + for r in range(sp): + t = ttnn.Tensor(host_t[r : r + 1], dtype) + if layout == ttnn.TILE_LAYOUT: + t = ttnn.to_layout(t, ttnn.TILE_LAYOUT) + parts.extend([t] * tp) + return ttnn.from_host_shards(parts, ttnn.MeshShape(sp, tp)) + + h = via_from_torch() + buf = ttnn.to_device(h, mesh_device) + rec = {"shape": list(shape), "dtype": str(dtype), "layout": str(layout)} + for label, build in (("from_torch", via_from_torch), ("shards", via_shards)): + builds, copies = [], [] + for i in range(70): + t0 = time.perf_counter() + h = build() + t1 = time.perf_counter() + ttnn.copy_host_to_device_tensor(h, buf) + t2 = time.perf_counter() + if i >= 10: + builds.append(t1 - t0) + copies.append(t2 - t1) + ttnn.synchronize_device(mesh_device) + rec[label] = {"build": _stats(builds), "copy": _stats(copies)} + syncs = [] + for _ in range(20): + t0 = time.perf_counter() + ttnn.synchronize_device(mesh_device) + syncs.append(time.perf_counter() - t0) + rec["sync"] = _stats(syncs) + out[name] = rec + print( + f"{name:26s} from_torch build {rec['from_torch']['build']['median_us']:7.0f} us copy " + f"{rec['from_torch']['copy']['median_us']:7.0f} us | shards build {rec['shards']['build']['median_us']:7.0f}" + f" us copy {rec['shards']['copy']['median_us']:7.0f} us | sync {rec['sync']['median_us']:6.0f} us" + ) + _dump("variants", out) + + +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(), indirect=True) +def test_v41_first_copy_after_step(mesh_device, device_params): + """A captured trace of ``V41_COPY_BENCH_OPS`` (default 200) small ops replayed non-blocking, a device + sync, a + read of a device tensor (as the step reads its ids), then four copies of the step's shapes: the first + copy's + cost against the others, with each of the replay, the sync and the read left out in turn.""" + sp, tp = tuple(mesh_device.shape) + mapper = ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(0, None)) + n_ops = int(os.environ.get("V41_COPY_BENCH_OPS", "200")) + x = ttnn.from_torch( + torch.randn(sp, 1, 32, 1024), + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + device=mesh_device, + mesh_mapper=mapper, + ) + + def graph(t): + for _ in range(n_ops): + t = ttnn.add(t, 1.0) + return t + + graph(x) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + y = graph(x) + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ids = ttnn.from_torch( + torch.zeros(sp, 1, 256, 32, dtype=torch.int32), + dtype=ttnn.uint32, + layout=ttnn.TILE_LAYOUT, + device=mesh_device, + mesh_mapper=mapper, + ) + shapes = [ + ("ints", torch.int32, ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 14, 32)), + ("floats", torch.float32, ttnn.bfloat16, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 2, 32)), + ("rope", torch.float32, ttnn.bfloat16, ttnn.TILE_LAYOUT, (1, 1, 32, 384)), + ("one", torch.int32, ttnn.uint32, ttnn.ROW_MAJOR_LAYOUT, (1, 1, 1, 12800)), + ] + hosts = { + name: ttnn.from_torch( + torch.zeros((sp,) + shape[1:], dtype=tdtype), dtype=dtype, layout=layout, mesh_mapper=mapper + ) + for name, tdtype, dtype, layout, shape in shapes + } + bufs = {name: ttnn.to_device(h, mesh_device) for name, h in hosts.items()} + out = {} + for case in ( + "replay+sync+read", + "replay+sync", + "sync+read", + "read", + "nothing", + "replay+sync+read, sync before copies", + ): + times = {name: [] for name in hosts} + replays = [] + for i in range(40): + t0 = time.perf_counter() + if "replay" in case: + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + if "sync" in case.split(",")[0]: + ttnn.synchronize_device(mesh_device) + t1 = time.perf_counter() + if "read" in case: + ttnn.to_torch(ttnn.get_device_tensors(ids)[0]) + if "sync before copies" in case: + ttnn.synchronize_device(mesh_device) + for name, h in hosts.items(): + t2 = time.perf_counter() + ttnn.copy_host_to_device_tensor(h, bufs[name]) + if i >= 5: + times[name].append(time.perf_counter() - t2) + if i >= 5: + replays.append(t1 - t0) + ttnn.synchronize_device(mesh_device) + rec = {name: _stats(v) for name, v in times.items()} + rec["replay_and_sync"] = _stats(replays) + out[case] = rec + print( + f"{case:40s} replay+sync {rec['replay_and_sync']['median_us']:7.0f} us | " + + " | ".join(f"{name} {rec[name]['median_us']:6.0f} (p90 {rec[name]['p90_us']:6.0f})" for name in hosts) + ) + ttnn.release_trace(mesh_device, tid) + _dump("firstcopy", out) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_indexer_perf.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_indexer_perf.py new file mode 100644 index 000000000000..05e9ae43d7d8 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_indexer_perf.py @@ -0,0 +1,335 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Long-context indexer performance (bead F10): measured time of ``TtV41Indexer`` at chunk start P. + +One 5120-token chunk at start P on LoudBox 2x4 with synthetic weights and index keys, for the three indexer roles: +layer 2 (ratio-2 index source), 20 (ratio-1 candidate source) and 24 (ratio-1 candidate index source, restricted to +layer 20's published candidate blocks). Warm (program cache hot) measurements: + +* ``traced_ms``: the whole forward (``scores`` + ``select``) captured in one trace and replayed (min over replays of + the blocking replay wall): device time, as the forward uploads and reads nothing from the host; +* ``stage_ms``: host wall time of the module's stages (``scores``, ``select``), synchronized only at stage + boundaries (dispatch pipelined inside a stage); +* ``ops``: every ttnn call synchronized individually (serialized; sums exceed the stage times), with the bytes each + output occupies per chip; outputs at least one score row wide (``[S/(sp*tp), >= T]``) are the [S, T] DRAM passes; +* ``model``: ``utils/v41_perf_model.py`` indexer ops (B9-B12) for the same workload. + +Logged as ``V41_INDEXER_PERF`` JSON lines, also appended to ``$V41_INDEXER_PERF_OUT`` (default +``generated/v41_indexer_perf.log``) so results survive a shared runner log being overwritten; no pass/fail bar +(the bead sets its targets from these numbers). +""" + +import json +import os +import time +import traceback +from collections import defaultdict + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.fabric_profiles import fabric2d_device_params +from models.demos.deepseek_v3_d_p.tt.v41.cache import V41ChunkTables +from models.demos.deepseek_v3_d_p.tt.v41.indexer import TtV41Indexer +from models.demos.deepseek_v3_d_p.utils import v41_perf_model as M + +CHUNK = 5120 +LAYERS = (2, 20, 24) +TIMED_ITERS = 2 +KEY_BLOCK_ROWS = 16384 # rows of the uploaded random index-key block (repeated on device over longer caches) +TTNN_OPS = ( + "from_torch", + "to_layout", + "slice", + "pad", + "add", + "multiply", + "subtract", + "maximum", + "minimum", + "max", + "where", + "eq", + "gt", + "gtz", + "typecast", + "bitcast", + "bitwise_and", + "abs", + "sign", + "floor", + "rsub", + "reshape", + "full", + "scatter", + "linear", + "concat", + "mesh_partition", + "embedding", + "gather", + "transpose", + "ge", + "bitwise_right_shift", + "bitwise_left_shift", + "repeat_interleave", + "generic_op", +) +EXPERIMENTAL_OPS = ( + "indexer_score_dsa", + "topk_large_indices", + "rotary_embedding_llama", + "nlp_create_qkv_heads", + "all_gather_async", + "reduce_scatter_minimal_async", +) +ELEMENT_BYTES = { + ttnn.bfloat16: 2, + ttnn.float32: 4, + ttnn.uint32: 4, + ttnn.int32: 4, + ttnn.uint16: 2, + ttnn.bfloat8_b: 1088 / 1024, +} + + +OUT = os.environ.get("V41_INDEXER_PERF_OUT", "generated/v41_indexer_perf.log") + + +def _log(msg: str) -> None: + logger.info(msg) + os.makedirs(os.path.dirname(OUT) or ".", exist_ok=True) + with open(OUT, "a") as f: + f.write(f"{time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} {msg}\n") + + +def _per_chip_shape(t) -> list[int]: + return list(ttnn.get_device_tensors(t)[0].shape) + + +def _bytes(t) -> float: + n = 1 + for d in _per_chip_shape(t): + n *= d + return n * ELEMENT_BYTES.get(t.dtype, 2) + + +class _OpProfiler: + """Replaces ttnn callables with device-synchronized timers for the duration of a ``with`` block; nested ttnn + calls inside a timed call are not timed separately. Records (stage, op, ms, output per-chip shape, bytes).""" + + def __init__(self, mesh_device): + self.mesh, self.stage, self.calls, self._depth, self._saved = mesh_device, "", [], 0, [] + self.captured = {} + + def _wrap(self, fn, name): + def timed(*args, **kwargs): + if self._depth: + return fn(*args, **kwargs) + self._depth += 1 + try: + ttnn.synchronize_device(self.mesh) + start = time.perf_counter() + out = fn(*args, **kwargs) + ttnn.synchronize_device(self.mesh) + ms = (time.perf_counter() - start) * 1e3 + finally: + self._depth -= 1 + if name == "indexer_score_dsa": + self.captured[name] = (args, kwargs) + first = out[0] if isinstance(out, (tuple, list)) else out + shape = _per_chip_shape(first) if isinstance(first, ttnn.Tensor) else [] + nbytes = _bytes(first) if isinstance(first, ttnn.Tensor) else 0.0 + self.calls.append((self.stage, name, ms, shape, nbytes)) + return out + + return timed + + def __enter__(self): + for owner, names in ((ttnn, TTNN_OPS), (ttnn.experimental, EXPERIMENTAL_OPS)): + for name in names: + fn = getattr(owner, name) + self._saved.append((owner, name, fn)) + setattr(owner, name, self._wrap(fn, name)) + return self + + def __exit__(self, *exc): + for owner, name, fn in reversed(self._saved): + setattr(owner, name, fn) + self._saved.clear() + + +def _model_ms(layer: int, start: int) -> dict: + ops = M.block_ops(layer, M.Workload(chunk=CHUNK, start=start), M.LOUDBOX_2X4) + ops = [o for o in ops if o.node in ("B9", "B10", "B11", "B12")] + e = M.compose(ops) + return { + "ops": {o.name: {"compute_ms": o.compute_ns / 1e6, "dram_ms": o.dram_ns / 1e6} for o in ops}, + "optimistic_ms": e.optimistic_ns / 1e6, + "conservative_ms": e.conservative_ns / 1e6, + } + + +def _random_rows(up, rows: int, width: int, g, block: int = KEY_BLOCK_ROWS): + """Replicated row-major bf16 [1, 1, rows, width]: one uploaded random block of ``block`` rows repeated on device + (a replicated host upload of a 0.5M-row cache outlasts the safe runner's dispatch timeout).""" + src = up(torch.randn(1, 1, min(rows, block), width, generator=g).to(torch.bfloat16), None, ttnn.ROW_MAJOR_LAYOUT) + if rows <= block: + return src + out = ttnn.allocate_tensor_on_device( + ttnn.Shape([1, 1, rows, width]), ttnn.bfloat16, ttnn.ROW_MAJOR_LAYOUT, src.device(), ttnn.DRAM_MEMORY_CONFIG + ) + for first in range(0, rows, block): + n = min(block, rows - first) + piece = src if n == block else ttnn.slice(src, [0, 0, 0, 0], [1, 1, n, width]) + ttnn.experimental.slice_write(piece, out, [0, 0, first, 0], [1, 1, first + n, width], [1, 1, 1, 1]) + return out + + +def _run(indexer, x, qr, index_k, tables, start, candidates, timer=None): + """Indexer forward split into its stages (same calls as ``TtV41Indexer.forward``) -> (idx, published, ms, ...).""" + ms = {} + + def stage(name, fn, *args): + if timer is not None: + timer.stage = name + ttnn.synchronize_device(indexer.mesh_device) + t0 = time.perf_counter() + out = fn(*args) + ttnn.synchronize_device(indexer.mesh_device) + ms[name] = (time.perf_counter() - t0) * 1e3 + return out + + score, visible = stage("scores", indexer.scores, x, qr, index_k, tables, start, CHUNK) + idx, published = stage("select", indexer.select, score, tables, start, visible, candidates) + ms["total"] = sum(ms.values()) + return idx, published, ms, score, visible + + +def _traced_ms(indexer, x, qr, index_k, tables, start, candidates, replays: int = 3) -> float: + """Min blocking replay wall of one captured forward (device time; nothing crosses the host inside it).""" + mesh = indexer.mesh_device + indexer(x, qr, index_k, tables, start, CHUNK, candidates) # compile + ttnn.synchronize_device(mesh) + tid = ttnn.begin_trace_capture(mesh, cq_id=0) + indexer(x, qr, index_k, tables, start, CHUNK, candidates) + ttnn.end_trace_capture(mesh, tid, cq_id=0) + ttnn.execute_trace(mesh, tid, cq_id=0, blocking=True) # warm + best = float("inf") + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh, tid, cq_id=0, blocking=True) + best = min(best, (time.perf_counter() - t0) * 1e3) + ttnn.release_trace(mesh, tid) + return best + + +@pytest.mark.timeout(5400) +@pytest.mark.parametrize("start", [0, 16384, 51200, 131072, 262144, 512000, 1048576], ids=lambda p: f"P{p}") +@pytest.mark.parametrize( + "mesh_device, device_params", + [ + pytest.param( + (2, 4), + fabric2d_device_params(trace_region_size=64 << 20), + marks=pytest.mark.requires_mesh_topology(mesh_shape=(2, 4), topology="mesh-2x4"), + id="fabric2d-mesh-2x4", + ) + ], + indirect=True, +) +def test_v41_indexer_perf(mesh_device, device_params, start): + shape, (sp, tp) = tuple(mesh_device.shape), tuple(mesh_device.shape) + g = torch.Generator().manual_seed(0) + t0 = time.perf_counter() + + def up(t, dims, layout=ttnn.TILE_LAYOUT): + mapper = ( + ttnn.ReplicateTensorToMesh(mesh_device) + if dims is None + else ttnn.ShardTensor2dMesh(mesh_device, shape, dims=dims) + ) + return ttnn.from_torch(t, device=mesh_device, dtype=ttnn.bfloat16, layout=layout, mesh_mapper=mapper) + + x = up(torch.randn(1, 1, CHUNK, C.EMB_SIZE, generator=g).to(torch.bfloat16), (2, 3)) + qr = up(torch.randn(1, 1, CHUNK, C.Q_LORA_RANK, generator=g).to(torch.bfloat16), (2, None)) + weights = { + "wq_b": torch.randn(C.INDEX_N_HEADS * C.INDEX_HEAD_DIM, C.Q_LORA_RANK, generator=g) * C.Q_LORA_RANK**-0.5, + "weights_proj": torch.randn(C.INDEX_N_HEADS, C.EMB_SIZE, generator=g) * C.EMB_SIZE**-0.5, + } + index_k = {} + for ratio in sorted({C.compress_ratio(l) for l in LAYERS}): + rows = -(-(start + CHUNK) // ratio // 32) * 32 + index_k[ratio] = _random_rows(up, rows, C.INDEX_HEAD_DIM, g) + _log(f"P={start}: inputs uploaded {time.perf_counter() - t0:.1f}s") + + try: + _measure(mesh_device, sp, tp, x, qr, weights, index_k, start, t0) + except Exception: + _log(f"P={start}: FAILED\n{traceback.format_exc()}") + raise + + +def _measure(mesh_device, sp, tp, x, qr, weights, index_k, start, t0): + candidates = None + tables = V41ChunkTables(mesh_device, C, start + CHUNK, CHUNK, list(LAYERS)) # position tables up to this chunk + for layer in LAYERS: + _log(f"P={start} L{layer}: start") + ratio = C.compress_ratio(layer) + indexer = TtV41Indexer(mesh_device, C, layer, weights) + cands = candidates if indexer.uses_candidates else None + t1 = time.perf_counter() + _, published, warm_ms, _, _ = _run(indexer, x, qr, index_k[ratio], tables, start, cands) + _log(f"P={start} L{layer}: warm-up {time.perf_counter() - t1:.1f}s {warm_ms}") + stage_ms = defaultdict(list) + for _ in range(TIMED_ITERS): + _, _, ms, _, _ = _run(indexer, x, qr, index_k[ratio], tables, start, cands) + for key, value in ms.items(): + stage_ms[key].append(value) + _log(f"P={start} L{layer}: stages {dict(stage_ms)}") + traced = _traced_ms(indexer, x, qr, index_k[ratio], tables, start, cands) + _log(f"P={start} L{layer}: traced {traced:.3f} ms") + + with _OpProfiler(mesh_device) as prof: + _, _, _, score, visible = _run(indexer, x, qr, index_k[ratio], tables, start, cands, timer=prof) + q_rows = CHUNK // (sp * tp) + width = score.shape[-1] + per_op = defaultdict(lambda: {"calls": 0, "ms": 0.0, "bytes": 0.0}) + st_passes, st_bytes = 0, 0.0 + for stage, name, ms, out_shape, nbytes in prof.calls: + key = f"{stage}.{name}" + per_op[key]["calls"] += 1 + per_op[key]["ms"] += ms + per_op[key]["bytes"] += nbytes + if len(out_shape) >= 2 and out_shape[-2] == q_rows and out_shape[-1] >= width - 64: + st_passes += 1 + st_bytes += nbytes + _log(f"P={start} L{layer}: op profile {len(prof.calls)} calls, [S,T] outputs {st_passes}") + uploads = [c for c in prof.calls if c[1] in ("from_torch", "full")] + + record = { + "start": start, + "layer": layer, + "ratio": ratio, + "visible": visible, + "score_width": width, + "q_rows_per_chip": q_rows, + "traced_ms": traced, + "stage_ms": {k: min(v) for k, v in stage_ms.items()}, + "stage_ms_all": dict(stage_ms), + "host_uploads_in_forward": len(uploads), + "ops": dict(sorted(per_op.items(), key=lambda kv: -kv[1]["ms"])), + "st_output_passes": st_passes, + "st_output_bytes_per_chip": st_bytes, + "score_unit_bytes_per_chip": q_rows * width * 2, + "model": _model_ms(layer, start), + } + _log("V41_INDEXER_PERF " + json.dumps(record)) + if indexer.is_candidate_source: + candidates = published + del score + _log(f"P={start}: done {time.perf_counter() - t0:.1f}s") diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_layout.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_layout.py new file mode 100644 index 000000000000..3eebd210a446 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_layout.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""Layout-agnostic V4.1 mesh checks (bead 11.1, no device): LoudBox 2x4 / 4x2 and Galaxy 8x4 / 4x8 accepted at +real and small dims, unsupported meshes and chunks rejected with the requirement they break.""" + +from types import SimpleNamespace + +import pytest + +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as Flash +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config as Small +from models.demos.deepseek_v3_d_p.tt.v41.cache import V41CacheGeometry +from models.demos.deepseek_v3_d_p.tt.v41.layout import V41MeshLayout + +LAYOUTS = [(2, 4), (4, 2), (8, 4), (4, 8)] + + +@pytest.mark.parametrize("config", [Flash, Small], ids=["flash", "small"]) +@pytest.mark.parametrize("shape", LAYOUTS, ids=[f"{s}x{t}" for s, t in LAYOUTS]) +def test_supported_layouts(shape, config): + layout = V41MeshLayout.of(SimpleNamespace(shape=shape)) + assert (layout.sp, layout.tp, layout.chips) == (*shape, shape[0] * shape[1]) + layout.check_model(config) + + +# smallest chunk each layout accepts: lcm(SP rule 2*32*sp, query rule 32*sp*tp) -> 256, 256, 1024, 1024; +# the production chunk 5120 gives 640 query rows per chip on 8 chips and 160 on 32 +@pytest.mark.parametrize( + "shape, smallest", [((2, 4), 256), ((4, 2), 256), ((8, 4), 1024), ((4, 8), 1024)], ids=lambda v: str(v) +) +def test_chunk_alignment(shape, smallest, expect_error): + layout = V41MeshLayout(*shape) + for chunk in (smallest, 5120): + V41CacheGeometry(Flash, max_seq_len=4 * 5120, chunk=chunk, sp=layout.sp) + layout.check_chunk(chunk) + below = smallest // 2 + with expect_error(AssertionError, "multiple of"): + V41CacheGeometry(Flash, max_seq_len=4 * 5120, chunk=below, sp=layout.sp) + layout.check_chunk(below) + + +def test_query_split_rejects_partial_tiles(expect_error): + # 2x4, chunk 128: the SP rule holds (64 rows per SP rank) but each chip would get 16 query rows + V41CacheGeometry(Flash, max_seq_len=1024, chunk=128, sp=2) + with expect_error(AssertionError, "32\\*sp\\*tp = 256"): + V41MeshLayout(2, 4).check_chunk(128) + + +class _Heads32Hidden2048(Flash): + EMB_SIZE = 2048 + NUM_ATTENTION_HEADS = 32 + + +class _Heads48(Flash): + NUM_ATTENTION_HEADS = 48 + + +@pytest.mark.parametrize( + "shape, config, message", + [ + ((1, 3), Flash, "hidden 5120 does not split over TP=3"), # 5120 / 3 is not whole tiles + ((1, 64), _Heads32Hidden2048, "32 attention heads do not split over TP=64"), # 2048 / 64 = 1 tile + ((1, 4), _Heads48, "multiple of 32 heads per chip, got 48"), # all heads per chip after head->sequence + ((4, 16), Flash, "8 output groups do not split over TP=16"), # 5120 / 16 = 10 tiles, 4 heads per chip + ((5, 4), Flash, "384 routed experts do not split over 20 chips"), + ], + ids=["hidden_tiles", "heads_over_tp", "sdpa_heads", "output_groups", "experts"], +) +def test_unsupported_layouts(shape, config, message, expect_error): + with expect_error(AssertionError, message): + V41MeshLayout(*shape).check_model(config) + + +def test_mesh_must_be_2d(expect_error): + with expect_error(AssertionError, "2D"): + V41MeshLayout.of(SimpleNamespace(shape=(32,))) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_mhc_decode_ops.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_mhc_decode_ops.py new file mode 100644 index 000000000000..126ac2851b10 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_mhc_decode_ops.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The mHC site's projection on the decode tile with two ops fewer (``tt/v41/mhc.py``, +``V41_MHC_PROJ_W=dram``: the +weight read from DRAM by the K-split cores, no L1 copy; ``V41_MHC_RSQRT=fused``: the rsqrt of the mean of +squares +applied to the mix's coefficient tile inside the stream mix, no ``ttnn.rsqrt``) against the +served path. + +No weights: a ``_V41Site`` with a random projection, ``V41_MD_TRIALS`` random fp32 ``[1, 1, 32, 4 * 1280]`` +stream +tiles a chip (every chip its own), ``fused_rms_project`` with each new choice alone and both compared bit for +bit +with the served path's on every chip, the prefill shape (``T`` 5,120 / sp) checked to keep the L1 copy, then +each +path under trace (``V41_MD_REPLAYS`` replays of ``V41_MD_CALLS`` calls) for the per-site device time. Results: +``V41_MD_RESULT``, ``V41_MD_OUT``.""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.tt_ccl import per_axis_topology +from models.demos.deepseek_v3_d_p.tt.v41.mhc import _V41Site, fused_rms_project, mhc_config + + +def _trace_us(mesh_device, fn, calls: int, replays: int) -> float: + for _ in range(2): + ttnn.deallocate(fn()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn() for _ in range(calls)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / calls + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + return round(best * 1e6, 1) + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=64 * 1024 * 1024), indirect=True) +def test_v41_mhc_decode_ops(mesh_device, device_params): + trials = int(os.environ.get("V41_MD_TRIALS", 4)) + replays = int(os.environ.get("V41_MD_REPLAYS", 20)) + calls = int(os.environ.get("V41_MD_CALLS", 32)) + sp, tp = tuple(mesh_device.shape) + cfg = mhc_config(C) + torch.manual_seed(119) + mix = (2 + cfg.n) * cfg.n + fn = torch.randn(mix, cfg.n * cfg.dim) * (cfg.n * cfg.dim) ** -0.5 + base, scale = torch.randn(mix) * 0.1, torch.tensor([1.0, 1.0, 1.0]) + site = _V41Site(mesh_device, cfg, fn, base, scale, tp_axis=1, topology=per_axis_topology()[1]) + tp_sum = site._tp_sum if site.tp_factor > 1 else None + local = cfg.n * cfg.dim // tp + + def project(x, weight_in_l1: bool, fused_rsqrt: bool): + return fused_rms_project( + x, site.fn_T, site.mix_hc, site.norm_eps, site.tp_factor, tp_sum, weight_in_l1, fused_rsqrt + ) + + def streams(rows: int, seed: int): + gen = torch.Generator().manual_seed(seed) + t = torch.randn(sp * rows, tp * local, generator=gen) * 2.0 # every chip its own hidden slice of every stream + return ttnn.from_torch( + t.view(1, 1, sp * rows, tp * local), + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(2, 3)), + ) + + variants = {"dram": (False, False), "rsqrt": (True, True), "both": (False, True)} + result = {"mesh": [sp, tp], "trials": []} + for trial in range(trials): + x = streams(32, trial) + ref = project(x, True, False) + for name, (w_l1, rs) in variants.items(): + out = project(x, w_l1, rs) + assert tuple(out.shape) == tuple(ref.shape) == (1, 1, 32, 32), (tuple(out.shape), tuple(ref.shape)) + diffs = [] + for a, b in zip(ttnn.get_device_tensors(ref), ttnn.get_device_tensors(out)): + a, b = ttnn.to_torch(a), ttnn.to_torch(b) + diffs.append(int((a[..., : site.mix_hc] != b[..., : site.mix_hc]).sum())) # the Sinkhorn's columns + entry = {"trial": trial, "variant": name, "words_differ": sum(diffs), "chips_differ": sum(map(bool, diffs))} + result["trials"].append(entry) + logger.info(f"mhc decode ops trial {json.dumps(entry)}") + assert sum(diffs) == 0, entry + ttnn.deallocate(out) + for t in (x, ref): + ttnn.deallocate(t) + # the prefill shapes (several tile rows) keep the L1 copy whatever the flag says: the chunk's and the + # short + # chunk's rows, each path under trace, the DRAM choice no slower than the served one + for rows in (5120 // sp, 2048 // sp): + big = streams(rows, 99) + out = project(big, False, True) + assert tuple(out.shape) == (1, 1, rows, 32), tuple(out.shape) + ttnn.deallocate(out) + served_us = _trace_us(mesh_device, lambda: project(big, True, False), 8, replays) + dram_us = _trace_us(mesh_device, lambda: project(big, False, True), 8, replays) + logger.info(f"mhc decode ops prefill rows {rows}: served {served_us} us a site, dram choice {dram_us}") + result[f"prefill_{rows}_us"] = {"served": served_us, "both": dram_us} + assert dram_us <= served_us * 1.05, (rows, served_us, dram_us) + ttnn.deallocate(big) + # the traced per-site time of each path + x = streams(32, 7) + timing = {"served": _trace_us(mesh_device, lambda: project(x, True, False), calls, replays)} + for name, (w_l1, rs) in variants.items(): + timing[name] = _trace_us(mesh_device, lambda: project(x, w_l1, rs), calls, replays) + for name, us in timing.items(): + logger.info(f"mhc decode ops traced {name}: {us} us a site (min of {replays} replays of {calls} calls)") + result["site_us"] = timing + logger.info(f"V41_MD_RESULT {json.dumps(result)}") + out_path = os.environ.get("V41_MD_OUT") + if out_path: + with open(os.path.expanduser(out_path), "w") as f: + json.dump(result, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_mhc_tp_sum.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_mhc_tp_sum.py new file mode 100644 index 000000000000..6e6e886f7ec5 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_mhc_tp_sum.py @@ -0,0 +1,170 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The mHC site's TP sum on the decode tile as one all-gather and a sum (``tt/v41/mhc.py``, +``V41_MHC_TP_SUM=gather``) +against ``TtMHCWrap._tp_sum`` (``all_reduce_async``: an all-broadcast and four layout ops around the same +sum). + +No weights: a ``_V41Site`` with a random projection, ``V41_TS_TRIALS`` random fp32 ``[1, 1, 32, 32]`` tiles a +chip +(every chip its own), the gathered sum compared bit for bit with the all-reduce's on every chip, then each +path +under trace (``V41_TS_REPLAYS`` replays of {calls} calls) for the per-site device time, and the prefill shape +(``T`` +5,120 / sp) checked to keep the all-reduce. Results: ``V41_TS_RESULT``, ``V41_TS_OUT``.""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tt.mhc.tt_mhc import TtMHCWrap +from models.demos.deepseek_v3_d_p.tt.tt_ccl import per_axis_topology +from models.demos.deepseek_v3_d_p.tt.v41 import mhc as mhc_mod +from models.demos.deepseek_v3_d_p.tt.v41.mhc import _V41Site, mhc_config + + +@pytest.mark.timeout(1800) +@pytest.mark.parametrize("mesh_device, device_params", galaxy_meshes(trace_region_size=64 * 1024 * 1024), indirect=True) +def test_v41_mhc_tp_sum(mesh_device, device_params): + mode = mhc_mod._TP_SUM + assert mode == "gather", "run with V41_MHC_TP_SUM=gather" + trials = int(os.environ.get("V41_TS_TRIALS", 4)) + replays = int(os.environ.get("V41_TS_REPLAYS", 20)) + calls = int(os.environ.get("V41_TS_CALLS", 32)) # calls a trace: the replay's fixed cost divided by this + sp, tp = tuple(mesh_device.shape) + cfg = mhc_config(C) + torch.manual_seed(11) + mix = (2 + cfg.n) * cfg.n + fn = torch.randn(mix, cfg.n * cfg.dim) * (cfg.n * cfg.dim) ** -0.5 + base, scale = torch.randn(mix) * 0.1, torch.tensor([1.0, 1.0, 1.0]) + site = _V41Site(mesh_device, cfg, fn, base, scale, tp_axis=1, topology=per_axis_topology()[1]) + composite = lambda t: TtMHCWrap._tp_sum(site, t) + + def tiles(rows: int, seed: int): + gen = torch.Generator().manual_seed(seed) + t = torch.randn(sp * rows, tp * 32, generator=gen) * 4.0 # [sp * rows, tp * 32]: every chip its own tile + return ttnn.from_torch( + t.view(1, 1, sp * rows, tp * 32), + device=mesh_device, + dtype=ttnn.float32, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(2, 3)), + ) + + result = {"mesh": [sp, tp], "mode": mode, "trials": []} + for trial in range(trials): + t = tiles(32, trial) + ref, out = composite(t), site._tp_sum(t) + assert tuple(out.shape) == tuple(ref.shape) == (1, 1, 32, 32), (tuple(out.shape), tuple(ref.shape)) + diffs = [] + for i, (a, b) in enumerate(zip(ttnn.get_device_tensors(ref), ttnn.get_device_tensors(out))): + a, b = ttnn.to_torch(a), ttnn.to_torch(b) + n = int((a != b).sum()) + diffs.append(n) + # the chips of one TP row read the same sum + rows = [ttnn.to_torch(c)[0, 0] for c in ttnn.get_device_tensors(out)] + same = all(torch.equal(rows[r * tp], rows[r * tp + c]) for r in range(sp) for c in range(tp)) + a, b = ( + ttnn.to_torch(ttnn.get_device_tensors(ref)[0]).double(), + ttnn.to_torch(ttnn.get_device_tensors(out)[0]).double(), + ) + rel = float((a - b).abs().max() / a.abs().max()) + entry = { + "trial": trial, + "words_differ": sum(diffs), + "chips_differ": sum(1 for d in diffs if d), + "tp_same": same, + "rel_max": rel, + } + result["trials"].append(entry) + logger.info(f"mhc tp sum trial {json.dumps(entry)}") + assert same and sum(diffs) == 0, entry # the composite's own reduction on the same tiles: bit for bit + for x in (t, ref, out): + ttnn.deallocate(x) + # the prefill shape keeps the all-reduce (the gather path is the decode tile's) + big = tiles(5120 // sp, 99) + out = site._tp_sum(big) + assert tuple(out.shape) == (1, 1, 5120 // sp, 32), tuple(out.shape) + for x in (big, out): + ttnn.deallocate(x) + # the traced per-site time of each path + t = tiles(32, 7) + timing = {} + for name, fn_ in (("all_reduce", lambda: composite(t)), (mode, lambda: site._tp_sum(t))): + for _ in range(2): + ttnn.deallocate(fn_()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn_() for _ in range(calls)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / calls + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + timing[name] = round(best * 1e6, 1) + logger.info(f"mhc tp sum traced {name}: {timing[name]} us a site (min of {replays} replays of {calls} calls)") + result["site_us"] = timing + # the pieces: each gather dim alone, the sum alone on a resident gathered tensor + ccl = site.tt_ccl + gather = lambda t, dim: ttnn.experimental.all_gather_async( + t, + dim=dim, + multi_device_global_semaphore=ccl.get_and_cycle_ag_semaphore_handles(cluster_axis=1), + barrier_semaphore=ccl.get_and_cycle_barrier_semaphore_handle(cluster_axis=1), + num_links=site.num_links, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + topology=site.topology, + cluster_axis=1, + ) + g1 = gather(t, 1) + pieces = { + "gather_dim1": lambda: gather(t, 1), + "gather_dim2": lambda: gather(t, 2), + "gather_dim3": lambda: gather(t, 3), + "sum_dim1": lambda: ttnn.sum(g1, dim=1, keepdim=True, memory_config=ttnn.DRAM_MEMORY_CONFIG), + "sum_dim1_nokeep": lambda: ttnn.sum(g1, dim=1, keepdim=False, memory_config=ttnn.DRAM_MEMORY_CONFIG), + } + piece_us = {} + for name, fn_ in pieces.items(): + for _ in range(2): + ttnn.deallocate(fn_()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn_() for _ in range(calls)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / calls + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + piece_us[name] = round(best * 1e6, 1) + logger.info(f"mhc tp sum piece {name}: {piece_us[name]} us (min of {replays} replays of {calls} calls)") + result["piece_us"] = piece_us + logger.info(f"V41_TS_RESULT {json.dumps(result)}") + out_path = os.environ.get("V41_TS_OUT") + if out_path: + with open(os.path.expanduser(out_path), "w") as f: + json.dump(result, f, indent=1) diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_moe_c1.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_moe_c1.py new file mode 100644 index 000000000000..4f156f49634f --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_moe_c1.py @@ -0,0 +1,531 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""The c1 decode MoE (``tt/v41/moe_c1.py``, ``V41_DECODE_C1_MOE=1``) against the batched step's 32-row MoE on +one +live row. + +One MoE layer (``V41_C1_LAYER``, default 2) from the weight cache, built as the batched decoder builds it (the +decode routing and tail on), fed ``V41_C1_TRIALS`` random tiles (default 8) with a random live (rank, row): +both +paths on the same input, the live row's output compared over the full hidden width (bf16 tolerance: the row's +bfp8 +packing and the fp32 order of the weighted sum are the only differences), the c1 path's idle rows zero. The +per-chip expert counts of every trial are read back so the run covers chips with 0, 1, 2, .. of the row's +experts. Then both paths under trace (``V41_C1_REPLAYS``, default 20): the per-layer device time. + +Run with ``V41_DECODE_ROUTING=1 V41_DECODE_TAIL=1 V41_DECODE_C1_MOE=1 TT_V41_EXPERT_DTYPE=bfp4`` (the served +expert dtype); with the served catalog's ``V41_SHARED_IN_FFN=rs`` the 32-row path folds the +shared expert's partial into its tail and the c1 path takes that partial from the tail's ``pending_shared`` +(``C1Moe.shared_fold`` reads ``pending``; ``returned`` with the MoE's own shared expert), and the test checks +the +tail holds nothing after either path. + +``V41_C1_ROWS=n`` (default 1): the c1 MoE on ``n`` live rows of one user (``C1Moe(rows=n)``, the speculative +verify +step's ``1 + K`` rows): rows ``u .. u + n - 1`` of the live rank against the 32-row path, the +idle rows zero, and one traced call replayed against the eager call on the same input (bit for bit).""" + +import json +import os +import time + +import pytest +import torch +from loguru import logger + +import ttnn +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41.galaxy_meshes import galaxy_meshes +from models.demos.deepseek_v3_d_p.tests.v41.weight_cache import weight_cache_dir +from models.demos.deepseek_v3_d_p.tt.v41.batch_decode import TtV41BatchDecoder +from models.demos.deepseek_v3_d_p.tt.v41.decode import ROWS +from models.demos.deepseek_v3_d_p.tt.v41.moe_c1 import C1Moe +from models.demos.deepseek_v3_d_p.tt.v41.transformer import TtV41Transformer +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint + +try: + from tracy import signpost +except ModuleNotFoundError: # not under the tracy wrapper + + def signpost(_name: str) -> None: + pass + + +def _pcc(a: torch.Tensor, b: torch.Tensor) -> float: + a, b = a.flatten().double(), b.flatten().double() + return float(torch.corrcoef(torch.stack([a, b]))[0, 1]) + + +def _oracle_row(moe, c1, x: torch.Tensor, experts: list, weights: torch.Tensor, tp: int) -> torch.Tensor: + """The MoE of one row in fp64 from the device's own weights (bfp4 experts, bfp8 shared expert, dequantized + by + ``to_torch``): ``sum_k w_k expert_k(x) + shared(x)``, ``x`` the experts' input row (after the FP8 QDQ).""" + routed = c1.routed + two_d = lambda t: t.reshape(*t.shape[-2:]).double() + hidden = x.shape[0] + limit = float( + moe.moe.shared_expert.clamped_silu_glu_limit + if hasattr(moe.moe.shared_expert, "clamped_silu_glu_limit") + else getattr(moe.moe.shared_expert, "_shared").clamped_silu_glu_limit + ) + + def ffn(g, u, d): + inter = d.shape[0] + gate, up = x @ g[:, :inter], x @ u[:, :inter] + act = torch.nn.functional.silu(torch.clamp(gate, max=limit)) * torch.clamp(up, -limit, limit) + return act @ d[:, :hidden] + + tables = [ttnn.to_torch(t).flatten().tolist() for t in ttnn.get_device_tensors(c1.table)] + out = torch.zeros(hidden, dtype=torch.float64) + for k, e in enumerate(experts): + chip = next(i for i, tab in enumerate(tables) if e in tab) + l = tables[chip].index(e) + g = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.gate_projs[l])[chip])) + u = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.up_projs[l])[chip])) + d = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.down_projs[l])[chip])) + out += weights[k] * ffn(g, u, d) + shared = getattr(moe.moe.shared_expert, "_shared", moe.moe.shared_expert) + for c in range(tp): # the shared expert's TP shards of SP rank 0 + g = two_d(ttnn.to_torch(ttnn.get_device_tensors(shared.gate_proj)[c])) + u = two_d(ttnn.to_torch(ttnn.get_device_tensors(shared.up_proj)[c])) + d = two_d(ttnn.to_torch(ttnn.get_device_tensors(shared.down_proj)[c])) + out += ffn(g, u, d) + return out + + +def _row(t, rank: int, u: int, tp: int) -> torch.Tensor: + """Row ``u`` of rank ``rank``'s tile, the TP shards concatenated -> ``[hidden]`` fp32.""" + chips = ttnn.get_device_tensors(t) + return torch.cat([ttnn.to_torch(chips[rank * tp + c])[0, 0, u].float() for c in range(tp)]) + + +@pytest.mark.timeout(3600) +@pytest.mark.parametrize( + "mesh_device, device_params", galaxy_meshes(trace_region_size=256 * 1024 * 1024), indirect=True +) +def test_v41_moe_c1(mesh_device, device_params): + assert os.environ.get("V41_DECODE_C1_MOE") == "1", "run with V41_DECODE_C1_MOE=1 (and the decode routing)" + layer = int(os.environ.get("V41_C1_LAYER", 2)) + trials = int(os.environ.get("V41_C1_TRIALS", 8)) + replays = int(os.environ.get("V41_C1_REPLAYS", 20)) + n_rows = int(os.environ.get("V41_C1_ROWS", 1)) + users = 4 + ckpt = resolve_checkpoint() + if ckpt is None: + pytest.skip("V4.1 checkpoint shards not downloaded") + sp, tp = tuple(mesh_device.shape) + max_seq = 1024 + spec = orc.real_spec((layer,), max_seq, checkpoint=ckpt.root) + root = weight_cache_dir(spec, mesh_device.shape) + top = ckpt.read(["embed.weight", "norm.weight", "head.weight"]) + t0 = time.time() + model = TtV41Transformer( + mesh_device, + C, + [layer], + lambda l, include_moe: (load_layer if include_moe else load_layer_dense)(ckpt, l), + top["embed.weight"], + top["norm.weight"], + top["head.weight"], + max_seq_len=max_seq, + chunk=1024, + weight_cache_path=root, + ) + decoder_cls = type("RowsDecoder", (TtV41BatchDecoder,), {"C1_ROWS": n_rows}) if n_rows > 1 else TtV41BatchDecoder + decoder = decoder_cls(model, root, users_per_rank=users, trace=False) + logger.info(f"c1 moe: layer {layer} built in {time.time() - t0:.1f} s; {n_rows} live rows a call") + # one prefill first, as every served or tested flow has: without it the decode tail's + # ``ttnn.reduce_scatter`` + # leaves a 512-byte L1 block at 914,176 after its first call and every later routed-expert FFN program + # (the + # 32-row one included) is rejected at compile against it + torch.manual_seed(3) + prompt = torch.randint(1000, 20000, (C.SLIDING_WINDOW + 7,)) + _, state = model.prefill(prompt, logit_positions=1) + del state # its device tensors stay allocated (DRAM); a unit test + moe, c1 = decoder.moes[0], decoder.c1_moes[0] + hidden = C.EMB_SIZE + torch.manual_seed(7) + shared_in_ffn = os.environ.get("V41_SHARED_IN_FFN", "0") + shared_mode = os.environ.get("V41_DECODE_C1_SHARED", "fold") + result = { + "layer": layer, + "trials": [], + "mesh": [sp, tp], + "shared_in_ffn": shared_in_ffn, + "c1_shared": shared_mode, + "c1_select": os.environ.get("V41_DECODE_C1_SELECT", "fresh"), + "c1_ffn": os.environ.get("V41_DECODE_C1_FFN", "composite"), + "c1_ffn_fidelity": os.environ.get("V41_DECODE_C1_FFN_FIDELITY", "lofi"), + "rows": n_rows, + } + + def host_rows(r: int, u: int) -> dict: + """``C1Moe.host_rows`` for ``n_rows`` rows ``u .. u + n_rows - 1`` of rank ``r`` (rows 0 .. n - 1 of + the + c1 tile and back; the verify decoder's layout).""" + if n_rows == 1: + return C1Moe.host_rows(sp, ROWS, (r, u)) + from models.demos.deepseek_v3_d_p.tt.v41.moe_c1 import _SENTINEL + + sel = torch.full((sp, ROWS), _SENTINEL, dtype=torch.int32) + put = torch.full((sp, ROWS), _SENTINEL, dtype=torch.int32) + for j in range(n_rows): + sel[r, j], put[r, u + j] = u + j, j + return {"c1sel": sel, "c1put": put} + + def rows_of(t, r: int, u: int) -> torch.Tensor: + return torch.stack([_row(t, r, u + j, tp) for j in range(n_rows)]) + + if shared_mode == "split": + # the split weights: every chip's piece bit for bit the host slice of its TP shard + split = c1.shared_split + shared = getattr(moe.moe.shared_expert, "_shared", moe.moe.shared_expert) + two_d = lambda t: t.reshape(*t.shape[-2:]) + for name, full, piece, axis in ( + ("gate", shared.gate_proj, split.gate, -1), + ("up", shared.up_proj, split.up, -1), + ("down", shared.down_proj, split.down, -2), + ): + full_chips, piece_chips = ttnn.get_device_tensors(full), ttnn.get_device_tensors(piece) + for i, (fc, pc) in enumerate(zip(full_chips, piece_chips)): + s_rank = i // tp + lo, n = ROWS * split.offsets[s_rank], ROWS * split.counts[s_rank] + want = torch.zeros((split.hidden, split.width) if axis == -1 else (split.width, split.hidden)) + src = two_d(ttnn.to_torch(fc).float()) + if axis == -1: + want[:, :n] = src[:, lo : lo + n] + else: + want[:n] = src[lo : lo + n] + got = two_d(ttnn.to_torch(pc).float()) + assert torch.equal(want, got), f"{name} split on chip {i}: {int((want != got).sum())} words differ" + result["split"] = { + "width": split.width, + "counts": split.counts, + "offsets": split.offsets, + "exact": True, + } + logger.info(f"c1 shared split: width {split.width}, tiles per SP rank {split.counts}, bit-exact on 32 chips") + worst = {"max_abs": 0.0, "pcc": 1.0, "rel": 0.0} + coverage = {} + use_oracle = os.environ.get("V41_C1_ORACLE", "1" if result["c1_ffn"] == "matvec" else "0") == "1" and n_rows == 1 + oracle_worst = {"full": 0.0, "c1": 0.0} + for trial in range(trials): + h = torch.randn(sp, ROWS, hidden) * 1.0 + tt_h = ttnn.from_torch( + h.view(1, 1, sp * ROWS, hidden), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(2, 3)), + ) + r, u = int(torch.randint(sp, (1,))), int(torch.randint(users - n_rows + 1, (1,))) + rows = {k: decoder.pool.per_rank(v.view(sp, 1, 1, ROWS), ttnn.uint32) for k, v in host_rows(r, u).items()} + sel, put = C1Moe.onehots(rows["c1sel"], rows["c1put"], decoder.c1_table) + c1.debug = {} + ref = decoder._ffn(moe, tt_h) + assert moe.moe.reduce_module.pending_shared is None, "the 32-row path left a shared partial in the tail" + out = c1(tt_h, sel, put, n_rows) + assert moe.moe.reduce_module.pending_shared is None, "the c1 path left a shared partial in the tail" + expect_fold = "split" if shared_mode == "split" else ("pending" if shared_in_ffn != "0" else "returned") + assert c1.shared_fold == expect_fold, (c1.shared_fold, expect_fold) + result["shared_fold"] = c1.shared_fold + if trial == trials - 1 and os.environ.get("V41_C1_PROFILE") == "1": + # With device profiling enabled: one eager call of each path between signposts (the per-op kernel split) + ttnn.synchronize_device(mesh_device) + signpost("start") + ttnn.deallocate(decoder._ffn(moe, tt_h)) + ttnn.synchronize_device(mesh_device) + signpost("c1") + ttnn.deallocate(c1(tt_h, sel, put, n_rows)) + ttnn.synchronize_device(mesh_device) + signpost("stop") + counts = [int(ttnn.to_torch(t).sum()) for t in ttnn.get_device_tensors(c1.debug["counts"])] + if c1.W0 is not None: + # the persistent weight tile against a fresh one from the same indices and weights, every chip + from models.demos.deepseek_v3_d_p.tt.v41.moe_c1 import c1_select + + fresh_counts, fresh_w = c1_select(c1.debug["indices"], c1.debug["weights"], c1.table, c1.E, rows=n_rows) + for i, (a, b) in enumerate(zip(ttnn.get_device_tensors(fresh_w), ttnn.get_device_tensors(c1.W0))): + assert torch.equal(ttnn.to_torch(a), ttnn.to_torch(b)), f"persistent W differs on chip {i}" + for i, (a, b) in enumerate( + zip(ttnn.get_device_tensors(fresh_counts), ttnn.get_device_tensors(c1.debug["counts"])) + ): + assert torch.equal(ttnn.to_torch(a), ttnn.to_torch(b)), f"counts differ on chip {i}" + ttnn.deallocate(fresh_counts) + ttnn.deallocate(fresh_w) + idx = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["indices"])[0])[0, :6].tolist() + ref_row, out_row = rows_of(ref, r, u), rows_of(out, r, u) + per_row = {} + if n_rows > 1: + # per row: the c1 row against the 32-row path's, and a second eager call on the same input against + # the first (determinism); the rows' expert ids and the chip counts + again = c1(tt_h, sel, put, n_rows) + again_rows = rows_of(again, r, u) + ttnn.deallocate(again) + all_idx = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["indices"])[0])[:n_rows, :6].tolist() + all_w = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["weights"])[0])[:n_rows, :6].double() + x_rows = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["x_full"])[0])[0, 0, :n_rows].double() + per_row = { + "pcc": [round(_pcc(ref_row[t], out_row[t]), 5) for t in range(n_rows)], + "max_abs": [round(float((ref_row[t] - out_row[t]).abs().max()), 4) for t in range(n_rows)], + "again_max_abs": [round(float((again_rows[t] - out_row[t]).abs().max()), 4) for t in range(n_rows)], + "experts": all_idx, + "oracle_pcc_full": [], + "oracle_pcc_c1": [], + } + for t in range(n_rows): # the fp64 MoE of each row from the device's own weights + exact = _oracle_row(moe, c1, x_rows[t], all_idx[t], all_w[t], tp) + per_row["oracle_pcc_full"].append(round(_pcc(exact.float(), ref_row[t]), 5)) + per_row["oracle_pcc_c1"].append(round(_pcc(exact.float(), out_row[t]), 5)) + # chip 0's pieces: W's nonzero entries per row, the FFN output rows of its regions against the + # oracle's + # expert FFN of the same row, the local weighted sum and the shared partial + W_host = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["W"])[0])[0, 0].float() + y_host = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["y"])[0])[0, 0].float() + table0 = ttnn.to_torch(ttnn.get_device_tensors(c1.table)[0]).flatten().tolist() + counts0 = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["counts"])[0]).flatten().tolist() + pieces = {"W_nonzero": [], "y_pcc": [], "counts_mine": [int(counts0[e]) for e in table0]} + routed = c1.routed + two_d = lambda tt: tt.reshape(*tt.shape[-2:]).double() + limit = float(getattr(moe.moe.shared_expert, "_shared", moe.moe.shared_expert).clamped_silu_glu_limit) + for t in range(n_rows): + nz = [(int(i), round(float(W_host[t, i]), 5)) for i in torch.nonzero(W_host[t]).flatten().tolist()] + pieces["W_nonzero"].append(nz) + row_pcc = [] + for l, e in enumerate(table0): + if e in all_idx[t]: + g = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.gate_projs[l])[0])) + u_ = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.up_projs[l])[0])) + d = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.down_projs[l])[0])) + x = x_rows[t] + inter = d.shape[0] + gate_, up_ = x @ g[:, :inter], x @ u_[:, :inter] + act = torch.nn.functional.silu(torch.clamp(gate_, max=limit)) * torch.clamp(up_, -limit, limit) + want = (act @ d[:, : x.shape[0]]).float() + row_pcc.append((l, e, round(_pcc(want, y_host[32 * l + t]), 5))) + pieces["y_pcc"].append(row_pcc) + # the local weighted sum against W @ y on the host, the shared piece against the split's own + # weights + part_host = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["part"])[0])[0, 0].float() + shared_host = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["shared"])[0])[0, 0].float() + want_part = W_host @ y_host + pieces["part_pcc"] = [round(_pcc(want_part[t], part_host[t]), 5) for t in range(n_rows)] + pieces["part_absmax"] = [round(float(part_host[t].abs().max()), 4) for t in range(n_rows)] + if c1.shared_split is not None: + sp_ = c1.shared_split + g0 = two_d(ttnn.to_torch(ttnn.get_device_tensors(sp_.gate)[0])) + u0 = two_d(ttnn.to_torch(ttnn.get_device_tensors(sp_.up)[0])) + d0 = two_d(ttnn.to_torch(ttnn.get_device_tensors(sp_.down)[0])) + pieces["shared_pcc"] = [] + for t in range(n_rows): + x = x_rows[t] + act = torch.nn.functional.silu(torch.clamp(x @ g0, max=limit)) * torch.clamp(x @ u0, -limit, limit) + pieces["shared_pcc"].append(round(_pcc((act @ d0).float(), shared_host[t]), 5)) + # the collectives: every chip's total (after the shared add) summed on the host against the TP + # reduce-scatter's and the SP gather-sum's rows on every chip + totals = [ttnn.to_torch(t)[0, 0].float() for t in ttnn.get_device_tensors(c1.debug["total"])] + tp_parts = [ttnn.to_torch(t)[0, 0].float() for t in ttnn.get_device_tensors(c1.debug["tp_part"])] + summeds = [ttnn.to_torch(t)[0, 0].float() for t in ttnn.get_device_tensors(c1.debug["summed"])] + width = tp_parts[0].shape[1] + pieces["rs_pcc"], pieces["sum_pcc"] = [], [] + for t in range(n_rows): + rs, sm = [], [] + for chip in range(len(totals)): + r_, c_ = divmod(chip, tp) + want_rs = sum(totals[r_ * tp + k][t, c_ * width : (c_ + 1) * width] for k in range(tp)) + rs.append(round(_pcc(want_rs, tp_parts[chip][t]), 4)) + want_sum = sum(totals[k][t, c_ * width : (c_ + 1) * width] for k in range(len(totals))) + sm.append(round(_pcc(want_sum, summeds[chip][t]), 4)) + pieces["rs_pcc"].append([min(rs), max(rs)]) + pieces["sum_pcc"].append([min(sm), max(sm)]) + # every chip's total against the host's: the row's experts on the chip (weights and FFN from the + # device weights) plus the chip's shared piece; the chips whose row is off are listed + tables = [ttnn.to_torch(t).flatten().tolist() for t in ttnn.get_device_tensors(c1.table)] + W_all = [ttnn.to_torch(t)[0, 0].float() for t in ttnn.get_device_tensors(c1.debug["W"])] + pieces["bad_chips"] = [] + if c1.shared_split is not None: + sp_ = c1.shared_split + for t in range(n_rows): + bad = [] + for chip, tab in enumerate(tables): + want = torch.zeros(x_rows[t].shape[0], dtype=torch.float64) + for l, e in enumerate(tab): + if e in all_idx[t]: + k_ = all_idx[t].index(e) + g = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.gate_projs[l])[chip])) + u_ = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.up_projs[l])[chip])) + d = two_d(ttnn.to_torch(ttnn.get_device_tensors(routed.down_projs[l])[chip])) + x = x_rows[t] + inter = d.shape[0] + act = torch.nn.functional.silu(torch.clamp(x @ g[:, :inter], max=limit)) * torch.clamp( + x @ u_[:, :inter], -limit, limit + ) + want += float(all_w[t][k_]) * (act @ d[:, : x.shape[0]]) + g0 = two_d(ttnn.to_torch(ttnn.get_device_tensors(sp_.gate)[chip])) + u0 = two_d(ttnn.to_torch(ttnn.get_device_tensors(sp_.up)[chip])) + d0 = two_d(ttnn.to_torch(ttnn.get_device_tensors(sp_.down)[chip])) + x = x_rows[t] + act = torch.nn.functional.silu(torch.clamp(x @ g0, max=limit)) * torch.clamp( + x @ u0, -limit, limit + ) + want += act @ d0 + p_ = _pcc(want.float(), totals[chip][t]) + if p_ < 0.999: + w_nz = [ + (int(i), round(float(W_all[chip][t, i]), 4)) + for i in torch.nonzero(W_all[chip][t]).flatten().tolist() + ] + mine = [(l, e) for l, e in enumerate(tab) if e in all_idx[t]] + bad.append({"chip": chip, "pcc": round(p_, 4), "experts": mine, "W": w_nz}) + pieces["bad_chips"].append(bad) + per_row["chip0"] = pieces + for k in ("W", "y", "part", "shared", "total", "tp_part", "summed"): + ttnn.deallocate(c1.debug[k]) + logger.info(f"c1 moe rows {n_rows} per row {json.dumps(per_row)}") + # the same call captured and replayed: the replay's rows must equal the eager call's (the verify + # step's rounds replay it) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + replayed = c1(tt_h, sel, put, n_rows) + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + eager_rows = rows_of(replayed, r, u) # the capture ran the ops once + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + replay_rows = rows_of(replayed, r, u) + ttnn.release_trace(mesh_device, tid) + ttnn.deallocate(replayed) + n_diff = int((eager_rows != replay_rows).sum()) + entry_replay = { + "captured_vs_eager_max_abs": float((eager_rows - out_row).abs().max()), + "replay_diff_words": n_diff, + } + logger.info(f"c1 moe rows {n_rows} replay check {entry_replay}") + result.setdefault("replay_diff_words", []).append(n_diff) + max_abs = float((ref_row - out_row).abs().max()) + rel = max_abs / max(float(ref_row.abs().max()), 1e-6) + pcc = _pcc(ref_row, out_row) + oracle = {} + if use_oracle: + # the fp64 MoE of the live row from the device's own (dequantized) weights: which path is nearer + # it + wts = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["weights"])[0])[0, :6].double() + x_full = ttnn.to_torch(ttnn.get_device_tensors(c1.debug["x_full"])[0])[0, 0, 0].double() + exact = _oracle_row(moe, c1, x_full, idx, wts, tp) + for name, row in (("full", ref_row), ("c1", out_row)): + d = float((exact - row.double()).abs().max()) + oracle[name] = { + "rel": round(d / max(float(exact.abs().max()), 1e-6), 6), + "pcc": round(_pcc(exact.float(), row), 6), + } + ttnn.deallocate(c1.debug["x_full"]) + elif "x_full" in c1.debug: + ttnn.deallocate(c1.debug["x_full"]) + ttnn.deallocate(c1.debug["weights"]) + c1.debug = None + # every other row of the c1 output is zero on every chip + idle_max = 0.0 + for i, chip in enumerate(ttnn.get_device_tensors(out)): + t = ttnn.to_torch(chip)[0, 0].float() + if i // tp == r: + t = torch.cat([t[:u], t[u + n_rows :]]) + idle_max = max(idle_max, float(t.abs().max())) + for n in counts: + coverage[n] = coverage.get(n, 0) + 1 + entry = { + "trial": trial, + "live": [r, u], + "experts": idx, + "chip_counts": counts, + "max_abs": round(max_abs, 5), + "rel": round(rel, 6), + "pcc": round(pcc, 6), + "ref_absmax": round(float(ref_row.abs().max()), 4), + "idle_max": idle_max, + "oracle": oracle, + "per_row": per_row, + } + result["trials"].append(entry) + logger.info(f"c1 moe trial {json.dumps(entry)}") + worst = { + "max_abs": max(worst["max_abs"], max_abs), + "pcc": min(worst["pcc"], pcc), + "rel": max(worst["rel"], rel), + } + for name in oracle: + oracle_worst[name] = max(oracle_worst[name], oracle[name]["rel"]) + if n_rows == 1: + assert sum(counts) == 6, f"the row's 6 experts over the chips: {counts}" + else: + assert sum(counts) % n_rows == 0 and 6 * n_rows <= sum(counts) <= 6 * n_rows * n_rows, counts + assert idle_max == 0.0, f"the c1 output's idle rows are not zero: {idle_max}" + for t in (ref, out, tt_h, sel, put): + ttnn.deallocate(t) + result["worst"] = {k: round(v, 6) for k, v in worst.items()} + if use_oracle: + result["oracle_worst_rel"] = oracle_worst + result["coverage"] = {str(k): v for k, v in sorted(coverage.items())} + # the per-layer device time of each path, under trace + h = torch.randn(sp, ROWS, hidden) + tt_h = ttnn.from_torch( + h.view(1, 1, sp * ROWS, hidden), + device=mesh_device, + dtype=ttnn.bfloat16, + layout=ttnn.TILE_LAYOUT, + memory_config=ttnn.DRAM_MEMORY_CONFIG, + mesh_mapper=ttnn.ShardTensor2dMesh(mesh_device, (sp, tp), dims=(2, 3)), + ) + rows = {k: decoder.pool.per_rank(v.view(sp, 1, 1, ROWS), ttnn.uint32) for k, v in host_rows(1, 0).items()} + sel, put = C1Moe.onehots(rows["c1sel"], rows["c1put"], decoder.c1_table) + c1.debug = {} + ttnn.deallocate(c1(tt_h, sel, put, n_rows)) + timing_counts = [int(ttnn.to_torch(t).sum()) for t in ttnn.get_device_tensors(c1.debug["counts"])] + for k in ("weights", "x_full"): + if k in c1.debug: + ttnn.deallocate(c1.debug[k]) + c1.debug = None + result["timing_chip_counts"] = timing_counts # the timed row's experts per chip (the busiest chip is the layer) + logger.info(f"c1 moe timing row: experts per chip {timing_counts}") + timing = {} + paths = (("full", lambda: decoder._ffn(moe, tt_h)), ("c1", lambda: c1(tt_h, sel, put, n_rows))) + for name, fn in () if os.environ.get("V41_C1_PROFILE") == "1" else paths: + for _ in range(2): + ttnn.deallocate(fn()) + ttnn.synchronize_device(mesh_device) + tid = ttnn.begin_trace_capture(mesh_device, cq_id=0) + outs = [fn() for _ in range(4)] + ttnn.end_trace_capture(mesh_device, tid, cq_id=0) + ttnn.synchronize_device(mesh_device) + best = None + for _ in range(replays): + t0 = time.perf_counter() + ttnn.execute_trace(mesh_device, tid, cq_id=0, blocking=False) + ttnn.synchronize_device(mesh_device) + dt = (time.perf_counter() - t0) / 4 + best = dt if best is None else min(best, dt) + ttnn.release_trace(mesh_device, tid) + for o in outs: + ttnn.deallocate(o) + timing[name] = round(best * 1e6, 1) + logger.info(f"c1 moe traced {name}: {timing[name]} us a layer (min of {replays} replays of 4 calls)") + result["layer_us"] = timing + logger.info(f"V41_C1_MOE_RESULT {json.dumps(result)}") + out_path = os.environ.get("V41_C1_OUT") + if out_path: + with open(os.path.expanduser(out_path), "w") as f: + json.dump(result, f, indent=1) + decoder.release() + assert not any( + result.get("replay_diff_words", []) + ), f"replayed calls differ from eager: {result['replay_diff_words']}" + assert worst["pcc"] > 0.999, worst + if use_oracle and worst["rel"] >= 0.02: + # a path nearer the exact MoE than the 32-row path may differ from that path by more than the bf16 bar + assert oracle_worst["c1"] <= oracle_worst["full"], oracle_worst + else: + assert worst["rel"] < 0.02, worst diff --git a/models/demos/deepseek_v3_d_p/tests/v41/test_v41_moe_placement.py b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_moe_placement.py new file mode 100644 index 000000000000..4e698e816189 --- /dev/null +++ b/models/demos/deepseek_v3_d_p/tests/v41/test_v41_moe_placement.py @@ -0,0 +1,423 @@ +# SPDX-FileCopyrightText: © 2026 Tenstorrent USA, Inc. + +# SPDX-License-Identifier: Apache-2.0 + +"""V4.1 routed-expert placement (bead 8y7.9.8, ``tt/v41/expert_placement.py``) on device. + +* ``test_v41_moe_placement_small``: ``TtV41Moe`` at SmallV41Config dims (synthetic weights) in the checkpoint + order and in a random expert order (the placement hook patched): the relabelling is invisible to the caller, + i.e. the gate logits and indices come back in checkpoint ids and the outputs match. +* ``test_v41_moe_placement_production``: ``TtV41Moe`` of every checkpoint layer (bfp8 routed experts, one + 5120-token chunk: the held-out prompts' reference MoE input ``ffn_in``, 2048 tokens tiled) with the placement the + code selects. Per layer and prompt: the device routing's per-chip routed pairs (most loaded chip / mean), top-6 + recall against the reference gate, a bit-identical repeat, and the traced MoE replay time. Logged as + ``V41_MOE_PLACEMENT`` JSON lines; under ``scripts/run_safe_pytest.sh --profile`` every measured replay is wrapped in + ``B moe L `` / ``E ...`` signposts for the per-chip op table. +* ``test_v41_moe_placement_text_ab`` (bead 8y7.9.12): same-tree A/B of the production placement (``profile``) against + the checkpoint order (``checkpoint``) on the G2 profile input, the first 5120 tokens of real text + (``oracle.text_tokens``) through the real sharing schedule (``test_v41_perf.text_stack``). Per layer, rounds + profile / checkpoint / checkpoint / profile (ABBA against drift) on the same inputs: the traced block replay (layer + span; its MoE swapped between placements) on the block input the device stack gives it, and the traced MoE replay + on the reference MoE input of that text (``text_spec``: 5120 distinct tokens, not tiled). Per placement: device + routing per-chip pairs, top-6 recall. Logged as ``V41_MOE_PLACEMENT_AB`` JSON lines; signposts + ``B block|moe L r`` under ``--profile``. The checkpoint order is a construction-time + choice of this test only (``expert_placement_choice``); its device weights live in their own weight-cache + directory, prepared outside the device lock with ``prepare_text_ab`` on a mock mesh. +""" + +import gc +import json +import time +from contextlib import contextmanager +from pathlib import Path + +import pytest +import torch +import torch.nn.functional as F +from loguru import logger +from tracy import signpost + +import ttnn +from models.common import timing_events +from models.demos.deepseek_v3_d_p.reference.deepseek_v41 import oracle as orc +from models.demos.deepseek_v3_d_p.reference.deepseek_v41_flash_config import DeepSeekV41FlashConfig as C +from models.demos.deepseek_v3_d_p.tests.v41 import expert_dtype_reference as R +from models.demos.deepseek_v3_d_p.tests.v41 import expert_load_profile as LP +from models.demos.deepseek_v3_d_p.tests.v41 import weight_cache +from models.demos.deepseek_v3_d_p.tests.v41.small_config import SmallV41Config +from models.demos.deepseek_v3_d_p.tests.v41.test_block_v41 import _pcc +from models.demos.deepseek_v3_d_p.tests.v41.test_moe_v41 import _run +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_expert_dtype import _weights +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_perf import text_stack +from models.demos.deepseek_v3_d_p.tests.v41.test_v41_trace import MESH, capture +from models.demos.deepseek_v3_d_p.tt.moe.init_helpers import get_sp_mesh_composer, get_tp_mesh_composer +from models.demos.deepseek_v3_d_p.tt.v41 import expert_placement +from models.demos.deepseek_v3_d_p.tt.v41 import moe as v41_moe +from models.demos.deepseek_v3_d_p.tt.v41.moe import TtV41Moe +from models.demos.deepseek_v3_d_p.tt.v41.weights import load_layer, load_layer_dense, resolve_checkpoint +from models.demos.deepseek_v3_d_p.utils.fast_cache_checker import init_checker + +SMALL_SEQ = 512 +TIMED_REPLAYS = 5 +RECALL_BAR = 0.95 # test_moe_v41 G1 top-6 recall bar +SMALL_LOGITS_PCC = 0.9999 # device gate logits (bf16) vs fp32 +SMALL_MARGIN = 0.005 # the permuted build may not be less accurate than the checkpoint-order build +PLACEMENTS = ("profile", "checkpoint") +AB_ROUNDS = ("profile", "checkpoint", "checkpoint", "profile") # per layer, ABBA: each placement in both halves + + +def text_spec() -> orc.OracleSpec: + """The reference of the G2 profile input: ``oracle.text_tokens(CHUNK)`` through the real sharing schedule.""" + return orc.real_spec(LP.LAYERS, R.CHUNK, checkpoint=orc.HF_SNAPSHOT) + + +@contextmanager +def expert_placement_choice(name: str): + """Construction-time expert order of this test: ``profile`` is the production placement + (``expert_placement.expert_order``); ``checkpoint`` keeps the checkpoint order (``expert_order`` patched to None, + and the weight-cache key built without the load profile, which then does not decide the stored bytes).""" + assert name in PLACEMENTS, name + with pytest.MonkeyPatch.context() as mp: + if name == "checkpoint": + mp.setattr(v41_moe, "expert_order", lambda *args: None) + mp.setattr(weight_cache, "CONVERSION_DATA", ()) + yield + + +def placement_cache_dir(name: str, mesh_shape) -> Path: + """The routed-expert bfp8 weight-cache directory of placement ``name`` (``profile``: the production one).""" + with expert_placement_choice(name): + root = weight_cache.weight_cache_dir(text_spec(), mesh_shape, ttnn.bfloat8_b) + return root if name == "profile" else root.with_name(root.name.replace("-mesh", "-checkpoint_order-mesh")) + + +def build_moe(mesh_device, ckpt, layer: int, name: str) -> TtV41Moe: + """``TtV41Moe`` of real ``layer`` (bfp8 routed experts, one CHUNK) in placement ``name``, as ``TtV41Block`` + builds it; the routed experts come from the placement's weight cache (built from the checkpoint on a miss).""" + root = placement_cache_dir(name, tuple(mesh_device.shape)) + root.mkdir(parents=True, exist_ok=True) + init_checker(root) + marker = root / f"layer_{layer}.complete" + start = time.perf_counter() + w = load_layer_dense(ckpt, layer) if marker.exists() else load_layer(ckpt, layer) + with expert_placement_choice(name): + moe = TtV41Moe( + mesh_device, C, layer, w, R.CHUNK, routed_expert_weights_dtype=ttnn.bfloat8_b, weight_cache_path=root + ) + marker.touch() + assert (moe.expert_order is None) == (name == "checkpoint"), (name, layer) + logger.info(f"MoE L{layer} {name} built {time.perf_counter() - start:.1f}s ({root.name})") + return moe + + +def prepare_text_ab(mesh_device) -> None: + """Fills both placements' weight caches of every layer (run on a mock mesh, outside the device lock).""" + ckpt = resolve_checkpoint() + for layer in LP.LAYERS: + for name in PLACEMENTS: + del_me = build_moe(mesh_device, ckpt, layer, name) + del del_me + gc.collect() + + +def _traced_ms(mesh_device, forward, moes, label: str) -> list[float]: + """Synchronized wall ms of TIMED_REPLAYS trace replays of ``forward`` (after a warm replay), each between + ``B